lvm.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864
  1. #
  2. # The Qubes OS Project, http://www.qubes-os.org
  3. #
  4. # Copyright (C) 2016 Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
  5. #
  6. # This library is free software; you can redistribute it and/or
  7. # modify it under the terms of the GNU Lesser General Public
  8. # License as published by the Free Software Foundation; either
  9. # version 2.1 of the License, or (at your option) any later version.
  10. #
  11. # This library is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. # Lesser General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU Lesser General Public
  17. # License along with this library; if not, see <https://www.gnu.org/licenses/>.
  18. #
  19. ''' Driver for storing vm images in a LVM thin pool '''
  20. import functools
  21. import logging
  22. import os
  23. import subprocess
  24. import time
  25. import asyncio
  26. import qubes
  27. import qubes.storage
  28. import qubes.utils
  29. def check_lvm_version():
  30. #Check if lvm is very very old, like in Travis-CI
  31. try:
  32. lvm_help = subprocess.check_output(['lvm', 'lvcreate', '--help'],
  33. stderr=subprocess.DEVNULL).decode()
  34. return '--setactivationskip' not in lvm_help
  35. except (subprocess.CalledProcessError, FileNotFoundError):
  36. pass
  37. lvm_is_very_old = check_lvm_version()
  38. class ThinPool(qubes.storage.Pool):
  39. ''' LVM Thin based pool implementation
  40. Volumes are stored as LVM thin volumes, in thin pool specified by
  41. *volume_group*/*thin_pool* arguments. LVM volume naming scheme:
  42. vm-{vm_name}-{volume_name}[-suffix]
  43. Where suffix can be one of:
  44. "-snap" - snapshot for currently running VM, at VM shutdown will be
  45. either discarded (if save_on_stop=False), or committed
  46. (if save_on_stop=True)
  47. "-{revision_id}" - volume revision - new revision is automatically
  48. created at each VM shutdown, *revisions_to_keep* control how many
  49. old revisions (in addition to the current one) should be stored
  50. "" (no suffix) - the most recent committed volume state; also volatile
  51. volume (snap_on_start=False, save_on_stop=False)
  52. On VM startup, new volume is created, depending on volume type,
  53. according to the table below:
  54. snap_on_start, save_on_stop
  55. False, False, - no suffix, fresh empty volume
  56. False, True, - "-snap", snapshot of last committed revision
  57. True , False, - "-snap", snapshot of last committed revision
  58. of source volume (from VM's template)
  59. True, True, - unsupported configuration
  60. Volume's revision_id format is "{timestamp}-back", where timestamp is in
  61. '%s' format (seconds since unix epoch)
  62. ''' # pylint: disable=protected-access
  63. size_cache = None
  64. driver = 'lvm_thin'
  65. def __init__(self, volume_group, thin_pool, revisions_to_keep=1, **kwargs):
  66. super(ThinPool, self).__init__(revisions_to_keep=revisions_to_keep,
  67. **kwargs)
  68. self.volume_group = volume_group
  69. self.thin_pool = thin_pool
  70. self._pool_id = "{!s}/{!s}".format(volume_group, thin_pool)
  71. self.log = logging.getLogger('qubes.storage.lvm.%s' % self._pool_id)
  72. self._volume_objects_cache = {}
  73. def __repr__(self):
  74. return '<{} at {:#x} name={!r} volume_group={!r} thin_pool={!r}>'.\
  75. format(
  76. type(self).__name__, id(self),
  77. self.name, self.volume_group, self.thin_pool)
  78. @property
  79. def config(self):
  80. return {
  81. 'name': self.name,
  82. 'volume_group': self.volume_group,
  83. 'thin_pool': self.thin_pool,
  84. 'driver': ThinPool.driver,
  85. 'revisions_to_keep': self.revisions_to_keep,
  86. }
  87. def destroy(self):
  88. pass # TODO Should we remove an existing pool?
  89. def init_volume(self, vm, volume_config):
  90. ''' Initialize a :py:class:`qubes.storage.Volume` from `volume_config`.
  91. '''
  92. if 'revisions_to_keep' not in volume_config.keys():
  93. volume_config['revisions_to_keep'] = self.revisions_to_keep
  94. if 'vid' not in volume_config.keys():
  95. if vm and hasattr(vm, 'name'):
  96. vm_name = vm.name
  97. else:
  98. # for the future if we have volumes not belonging to a vm
  99. vm_name = qubes.utils.random_string()
  100. assert self.name
  101. volume_config['vid'] = "{!s}/vm-{!s}-{!s}".format(
  102. self.volume_group, vm_name, volume_config['name'])
  103. volume_config['volume_group'] = self.volume_group
  104. volume_config['pool'] = self
  105. volume = ThinVolume(**volume_config)
  106. self._volume_objects_cache[volume_config['vid']] = volume
  107. return volume
  108. def setup(self):
  109. reset_cache()
  110. cache_key = self.volume_group + '/' + self.thin_pool
  111. if cache_key not in size_cache:
  112. raise qubes.storage.StoragePoolException(
  113. 'Thin pool {} does not exist'.format(cache_key))
  114. if size_cache[cache_key]['attr'][0] != 't':
  115. raise qubes.storage.StoragePoolException(
  116. 'Volume {} is not a thin pool'.format(cache_key))
  117. # TODO Should we create a non existing pool?
  118. def get_volume(self, vid):
  119. ''' Return a volume with given vid'''
  120. if vid in self._volume_objects_cache:
  121. return self._volume_objects_cache[vid]
  122. config = {
  123. 'pool': self,
  124. 'vid': vid,
  125. 'name': vid,
  126. 'volume_group': self.volume_group,
  127. }
  128. # don't cache this object, as it doesn't carry full configuration
  129. return ThinVolume(**config)
  130. def list_volumes(self):
  131. ''' Return a list of volumes managed by this pool '''
  132. volumes = []
  133. for vid, vol_info in size_cache.items():
  134. if not vid.startswith(self.volume_group + '/'):
  135. continue
  136. if vol_info['pool_lv'] != self.thin_pool:
  137. continue
  138. if vid.endswith('-snap') or vid.endswith('-import'):
  139. # implementation detail volume
  140. continue
  141. if vid.endswith('-back'):
  142. # old revisions
  143. continue
  144. volume = self.get_volume(vid)
  145. if volume in volumes:
  146. continue
  147. volumes.append(volume)
  148. return volumes
  149. @property
  150. def size(self):
  151. try:
  152. return qubes.storage.lvm.size_cache[
  153. self.volume_group + '/' + self.thin_pool]['size']
  154. except KeyError:
  155. return 0
  156. @property
  157. def usage(self):
  158. refresh_cache()
  159. try:
  160. return qubes.storage.lvm.size_cache[
  161. self.volume_group + '/' + self.thin_pool]['usage']
  162. except KeyError:
  163. return 0
  164. @property
  165. def usage_details(self):
  166. result = {}
  167. result['data_size'] = self.size
  168. result['data_usage'] = self.usage
  169. try:
  170. metadata_size = qubes.storage.lvm.size_cache[
  171. self.volume_group + '/' + self.thin_pool]['metadata_size']
  172. metadata_usage = qubes.storage.lvm.size_cache[
  173. self.volume_group + '/' + self.thin_pool]['metadata_usage']
  174. except KeyError:
  175. metadata_size = 0
  176. metadata_usage = 0
  177. result['metadata_size'] = metadata_size
  178. result['metadata_usage'] = metadata_usage
  179. return result
  180. _init_cache_cmd = ['lvs', '--noheadings', '-o',
  181. 'vg_name,pool_lv,name,lv_size,data_percent,lv_attr,origin,lv_metadata_size,'
  182. 'metadata_percent', '--units', 'b', '--separator', ';']
  183. def _parse_lvm_cache(lvm_output):
  184. result = {}
  185. for line in lvm_output.splitlines():
  186. line = line.decode().strip()
  187. pool_name, pool_lv, name, size, usage_percent, attr, \
  188. origin, metadata_size, metadata_percent = line.split(';', 8)
  189. if '' in [pool_name, name, size, usage_percent]:
  190. continue
  191. name = pool_name + "/" + name
  192. size = int(size[:-1]) # Remove 'B' suffix
  193. usage = int(size / 100 * float(usage_percent))
  194. if metadata_size:
  195. metadata_size = int(metadata_size[:-1])
  196. metadata_usage = int(metadata_size / 100 * float(metadata_percent))
  197. else:
  198. metadata_usage = None
  199. result[name] = {'size': size, 'usage': usage, 'pool_lv': pool_lv,
  200. 'attr': attr, 'origin': origin, 'metadata_size': metadata_size,
  201. 'metadata_usage': metadata_usage}
  202. return result
  203. def init_cache(log=logging.getLogger('qubes.storage.lvm')):
  204. cmd = _init_cache_cmd
  205. if os.getuid() != 0:
  206. cmd = ['sudo'] + cmd
  207. environ = os.environ.copy()
  208. environ['LC_ALL'] = 'C.utf8'
  209. p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  210. close_fds=True, env=environ)
  211. out, err = p.communicate()
  212. return_code = p.returncode
  213. if return_code == 0 and err:
  214. log.warning(err)
  215. elif return_code != 0:
  216. raise qubes.storage.StoragePoolException(err)
  217. return _parse_lvm_cache(out)
  218. @asyncio.coroutine
  219. def init_cache_coro(log=logging.getLogger('qubes.storage.lvm')):
  220. cmd = _init_cache_cmd
  221. if os.getuid() != 0:
  222. cmd = ['sudo'] + cmd
  223. environ = os.environ.copy()
  224. environ['LC_ALL'] = 'C.utf8'
  225. p = yield from asyncio.create_subprocess_exec(*cmd,
  226. stdout=subprocess.PIPE,
  227. stderr=subprocess.PIPE,
  228. close_fds=True, env=environ)
  229. out, err = yield from p.communicate()
  230. return_code = p.returncode
  231. if return_code == 0 and err:
  232. log.warning(err)
  233. elif return_code != 0:
  234. raise qubes.storage.StoragePoolException(err)
  235. return _parse_lvm_cache(out)
  236. size_cache_time = 0
  237. size_cache = init_cache()
  238. def _revision_sort_key(revision):
  239. '''Sort key for revisions. Sort them by time
  240. :returns timestamp
  241. '''
  242. if isinstance(revision, tuple):
  243. revision = revision[0]
  244. if '-' in revision:
  245. revision = revision.split('-')[0]
  246. return int(revision)
  247. def locked(method):
  248. '''Decorator running given Volume's coroutine under a lock.
  249. Needs to be added after wrapping with @asyncio.coroutine, for example:
  250. >>>@locked
  251. >>>@asyncio.coroutine
  252. >>>def start(self):
  253. >>> pass
  254. '''
  255. @asyncio.coroutine
  256. @functools.wraps(method)
  257. def wrapper(self, *args, **kwargs):
  258. with (yield from self._lock): # pylint: disable=protected-access
  259. return (yield from method(self, *args, **kwargs))
  260. return wrapper
  261. class ThinVolume(qubes.storage.Volume):
  262. ''' Default LVM thin volume implementation
  263. ''' # pylint: disable=too-few-public-methods
  264. def __init__(self, volume_group, **kwargs):
  265. self.volume_group = volume_group
  266. super(ThinVolume, self).__init__(**kwargs)
  267. self.log = logging.getLogger('qubes.storage.lvm.%s' % str(self.pool))
  268. if self.snap_on_start or self.save_on_stop:
  269. self._vid_snap = self.vid + '-snap'
  270. if self.save_on_stop:
  271. self._vid_import = self.vid + '-import'
  272. self._lock = asyncio.Lock()
  273. @property
  274. def path(self):
  275. return '/dev/' + self._vid_current
  276. @property
  277. def _vid_current(self):
  278. if self.vid in size_cache:
  279. return self.vid
  280. vol_revisions = self.revisions
  281. if vol_revisions:
  282. last_revision = \
  283. max(vol_revisions.items(), key=_revision_sort_key)[0]
  284. return self.vid + '-' + last_revision
  285. # detached pool? return expected path
  286. return self.vid
  287. @property
  288. def revisions(self):
  289. name_prefix = self.vid + '-'
  290. revisions = {}
  291. for revision_vid in size_cache:
  292. if not revision_vid.startswith(name_prefix):
  293. continue
  294. if not revision_vid.endswith('-back'):
  295. continue
  296. revision_vid = revision_vid[len(name_prefix):]
  297. if revision_vid.count('-') > 1:
  298. # VM+volume name is a prefix of another VM, see #4680
  299. continue
  300. # get revision without suffix
  301. seconds = int(revision_vid.split('-')[0])
  302. iso_date = qubes.storage.isodate(seconds).split('.', 1)[0]
  303. revisions[revision_vid] = iso_date
  304. return revisions
  305. @property
  306. def size(self):
  307. try:
  308. if self.is_dirty():
  309. return qubes.storage.lvm.size_cache[self._vid_snap]['size']
  310. return qubes.storage.lvm.size_cache[self._vid_current]['size']
  311. except KeyError:
  312. return self._size
  313. @size.setter
  314. def size(self, _):
  315. raise qubes.storage.StoragePoolException(
  316. "You shouldn't use lvm size setter")
  317. @asyncio.coroutine
  318. def _reset(self):
  319. ''' Resets a volatile volume '''
  320. assert not self.snap_on_start and not self.save_on_stop, \
  321. "Not a volatile volume"
  322. self.log.debug('Resetting volatile %s', self.vid)
  323. try:
  324. cmd = ['remove', self.vid]
  325. yield from qubes_lvm_coro(cmd, self.log)
  326. except qubes.storage.StoragePoolException:
  327. pass
  328. # pylint: disable=protected-access
  329. cmd = ['create', self.pool._pool_id, self.vid.split('/')[1],
  330. str(self.size)]
  331. yield from qubes_lvm_coro(cmd, self.log)
  332. @asyncio.coroutine
  333. def _remove_revisions(self, revisions=None):
  334. '''Remove old volume revisions.
  335. If no revisions list is given, it removes old revisions according to
  336. :py:attr:`revisions_to_keep`
  337. :param revisions: list of revisions to remove
  338. '''
  339. if revisions is None:
  340. revisions = sorted(self.revisions.items(),
  341. key=_revision_sort_key)
  342. # pylint: disable=invalid-unary-operand-type
  343. revisions = revisions[:(-self.revisions_to_keep) or None]
  344. revisions = [rev_id for rev_id, _ in revisions]
  345. for rev_id in revisions:
  346. # safety check
  347. assert rev_id != self._vid_current
  348. try:
  349. cmd = ['remove', self.vid + '-' + rev_id]
  350. yield from qubes_lvm_coro(cmd, self.log)
  351. except qubes.storage.StoragePoolException:
  352. pass
  353. @asyncio.coroutine
  354. def _commit(self, vid_to_commit=None, keep=False):
  355. '''
  356. Commit temporary volume into current one. By default
  357. :py:attr:`_vid_snap` is used (which is created by :py:meth:`start()`),
  358. but can be overriden by *vid_to_commit* argument.
  359. :param vid_to_commit: LVM volume ID to commit into this one
  360. :param keep: whether to keep or not *vid_to_commit*.
  361. IOW use 'clone' or 'rename' methods.
  362. :return: None
  363. '''
  364. msg = "Trying to commit {!s}, but it has save_on_stop == False"
  365. msg = msg.format(self)
  366. assert self.save_on_stop, msg
  367. msg = "Trying to commit {!s}, but it has rw == False"
  368. msg = msg.format(self)
  369. assert self.rw, msg
  370. if vid_to_commit is None:
  371. assert hasattr(self, '_vid_snap')
  372. vid_to_commit = self._vid_snap
  373. assert self._lock.locked()
  374. if not os.path.exists('/dev/' + vid_to_commit):
  375. # nothing to commit
  376. return
  377. if self._vid_current == self.vid:
  378. cmd = ['rename', self.vid,
  379. '{}-{}-back'.format(self.vid, int(time.time()))]
  380. yield from qubes_lvm_coro(cmd, self.log)
  381. yield from reset_cache_coro()
  382. cmd = ['clone' if keep else 'rename',
  383. vid_to_commit,
  384. self.vid]
  385. yield from qubes_lvm_coro(cmd, self.log)
  386. yield from reset_cache_coro()
  387. # make sure the one we've committed right now is properly
  388. # detected as the current one - before removing anything
  389. assert self._vid_current == self.vid
  390. # and remove old snapshots, if needed
  391. yield from self._remove_revisions()
  392. @locked
  393. @asyncio.coroutine
  394. def create(self):
  395. assert self.vid
  396. assert self.size
  397. if self.save_on_stop:
  398. if self.source:
  399. cmd = ['clone', self.source.path, self.vid]
  400. else:
  401. cmd = [
  402. 'create',
  403. self.pool._pool_id, # pylint: disable=protected-access
  404. self.vid.split('/', 1)[1],
  405. str(self.size)
  406. ]
  407. yield from qubes_lvm_coro(cmd, self.log)
  408. yield from reset_cache_coro()
  409. return self
  410. @locked
  411. @asyncio.coroutine
  412. def remove(self):
  413. assert self.vid
  414. try:
  415. if os.path.exists('/dev/' + self._vid_snap):
  416. cmd = ['remove', self._vid_snap]
  417. yield from qubes_lvm_coro(cmd, self.log)
  418. except AttributeError:
  419. pass
  420. try:
  421. if os.path.exists('/dev/' + self._vid_import):
  422. cmd = ['remove', self._vid_import]
  423. yield from qubes_lvm_coro(cmd, self.log)
  424. except AttributeError:
  425. pass
  426. yield from self._remove_revisions(self.revisions.keys())
  427. if not os.path.exists(self.path):
  428. return
  429. cmd = ['remove', self.path]
  430. yield from qubes_lvm_coro(cmd, self.log)
  431. yield from reset_cache_coro()
  432. # pylint: disable=protected-access
  433. self.pool._volume_objects_cache.pop(self.vid, None)
  434. def export(self):
  435. ''' Returns an object that can be `open()`. '''
  436. # make sure the device node is available
  437. qubes_lvm(['activate', self.path], self.log)
  438. devpath = self.path
  439. return devpath
  440. @locked
  441. @asyncio.coroutine
  442. def import_volume(self, src_volume):
  443. if not src_volume.save_on_stop:
  444. return self
  445. if self.is_dirty():
  446. raise qubes.storage.StoragePoolException(
  447. 'Cannot import to dirty volume {} -'
  448. ' start and stop a qube to cleanup'.format(self.vid))
  449. self.abort_if_import_in_progress()
  450. # HACK: neat trick to speed up testing if you have same physical thin
  451. # pool assigned to two qubes-pools i.e: qubes_dom0 and test-lvm
  452. # pylint: disable=line-too-long
  453. if isinstance(src_volume.pool, ThinPool) and \
  454. src_volume.pool.thin_pool == self.pool.thin_pool: # NOQA
  455. yield from self._commit(src_volume.path[len('/dev/'):], keep=True)
  456. else:
  457. cmd = ['create',
  458. self.pool._pool_id, # pylint: disable=protected-access
  459. self._vid_import.split('/')[1],
  460. str(src_volume.size)]
  461. yield from qubes_lvm_coro(cmd, self.log)
  462. src_path = src_volume.export()
  463. cmd = ['dd', 'if=' + src_path, 'of=/dev/' + self._vid_import,
  464. 'conv=sparse', 'status=none', 'bs=128K']
  465. if not os.access('/dev/' + self._vid_import, os.W_OK) or \
  466. not os.access(src_path, os.R_OK):
  467. cmd.insert(0, 'sudo')
  468. p = yield from asyncio.create_subprocess_exec(*cmd)
  469. yield from p.wait()
  470. if p.returncode != 0:
  471. cmd = ['remove', self._vid_import]
  472. yield from qubes_lvm_coro(cmd, self.log)
  473. raise qubes.storage.StoragePoolException(
  474. 'Failed to import volume {!r}, dd exit code: {}'.format(
  475. src_volume, p.returncode))
  476. yield from self._commit(self._vid_import)
  477. return self
  478. @locked
  479. @asyncio.coroutine
  480. def import_data(self, size):
  481. ''' Returns an object that can be `open()`. '''
  482. if self.is_dirty():
  483. raise qubes.storage.StoragePoolException(
  484. 'Cannot import data to dirty volume {}, stop the qube first'.
  485. format(self.vid))
  486. self.abort_if_import_in_progress()
  487. # pylint: disable=protected-access
  488. cmd = ['create', self.pool._pool_id, self._vid_import.split('/')[1],
  489. str(size)]
  490. yield from qubes_lvm_coro(cmd, self.log)
  491. yield from reset_cache_coro()
  492. devpath = '/dev/' + self._vid_import
  493. return devpath
  494. @locked
  495. @asyncio.coroutine
  496. def import_data_end(self, success):
  497. '''Either commit imported data, or discard temporary volume'''
  498. if not os.path.exists('/dev/' + self._vid_import):
  499. raise qubes.storage.StoragePoolException(
  500. 'No import operation in progress on {}'.format(self.vid))
  501. if success:
  502. yield from self._commit(self._vid_import)
  503. else:
  504. cmd = ['remove', self._vid_import]
  505. yield from qubes_lvm_coro(cmd, self.log)
  506. def abort_if_import_in_progress(self):
  507. try:
  508. devpath = '/dev/' + self._vid_import
  509. if os.path.exists(devpath):
  510. raise qubes.storage.StoragePoolException(
  511. 'Import operation in progress on {}'.format(self.vid))
  512. except AttributeError: # self._vid_import
  513. # no vid_import - import definitely not in progress
  514. pass
  515. def is_dirty(self):
  516. if self.save_on_stop:
  517. return os.path.exists('/dev/' + self._vid_snap)
  518. return False
  519. def is_outdated(self):
  520. if not self.snap_on_start:
  521. return False
  522. if self._vid_snap not in size_cache:
  523. return False
  524. return (size_cache[self._vid_snap]['origin'] !=
  525. self.source.path.split('/')[-1])
  526. @locked
  527. @asyncio.coroutine
  528. def revert(self, revision=None):
  529. if self.is_dirty():
  530. raise qubes.storage.StoragePoolException(
  531. 'Cannot revert dirty volume {}, stop the qube first'.format(
  532. self.vid))
  533. self.abort_if_import_in_progress()
  534. if revision is None:
  535. revision = \
  536. max(self.revisions.items(), key=_revision_sort_key)[0]
  537. old_path = '/dev/' + self.vid + '-' + revision
  538. if not os.path.exists(old_path):
  539. msg = "Volume {!s} has no {!s}".format(self, old_path)
  540. raise qubes.storage.StoragePoolException(msg)
  541. if self.vid in size_cache:
  542. cmd = ['remove', self.vid]
  543. yield from qubes_lvm_coro(cmd, self.log)
  544. cmd = ['clone', self.vid + '-' + revision, self.vid]
  545. yield from qubes_lvm_coro(cmd, self.log)
  546. yield from reset_cache_coro()
  547. return self
  548. @locked
  549. @asyncio.coroutine
  550. def resize(self, size):
  551. ''' Expands volume, throws
  552. :py:class:`qubst.storage.qubes.storage.StoragePoolException` if
  553. given size is less than current_size
  554. '''
  555. if not self.rw:
  556. msg = 'Can not resize reađonly volume {!s}'.format(self)
  557. raise qubes.storage.StoragePoolException(msg)
  558. if size < self.size:
  559. raise qubes.storage.StoragePoolException(
  560. 'For your own safety, shrinking of %s is'
  561. ' disabled (%d < %d). If you really know what you'
  562. ' are doing, use `lvresize` on %s manually.' %
  563. (self.name, size, self.size, self.vid))
  564. if size == self.size:
  565. return
  566. if self.is_dirty():
  567. cmd = ['extend', self._vid_snap, str(size)]
  568. yield from qubes_lvm_coro(cmd, self.log)
  569. elif hasattr(self, '_vid_import') and \
  570. os.path.exists('/dev/' + self._vid_import):
  571. cmd = ['extend', self._vid_import, str(size)]
  572. yield from qubes_lvm_coro(cmd, self.log)
  573. elif self.save_on_stop and not self.snap_on_start:
  574. cmd = ['extend', self._vid_current, str(size)]
  575. yield from qubes_lvm_coro(cmd, self.log)
  576. self._size = size
  577. yield from reset_cache_coro()
  578. @asyncio.coroutine
  579. def _snapshot(self):
  580. try:
  581. cmd = ['remove', self._vid_snap]
  582. yield from qubes_lvm_coro(cmd, self.log)
  583. except: # pylint: disable=bare-except
  584. pass
  585. if self.source is None:
  586. cmd = ['clone', self._vid_current, self._vid_snap]
  587. else:
  588. cmd = ['clone', self.source.path, self._vid_snap]
  589. yield from qubes_lvm_coro(cmd, self.log)
  590. @locked
  591. @asyncio.coroutine
  592. def start(self):
  593. self.abort_if_import_in_progress()
  594. try:
  595. if self.snap_on_start or self.save_on_stop:
  596. if not self.save_on_stop or not self.is_dirty():
  597. yield from self._snapshot()
  598. else:
  599. yield from self._reset()
  600. finally:
  601. yield from reset_cache_coro()
  602. return self
  603. @locked
  604. @asyncio.coroutine
  605. def stop(self):
  606. try:
  607. if self.save_on_stop:
  608. yield from self._commit()
  609. if self.snap_on_start and not self.save_on_stop:
  610. cmd = ['remove', self._vid_snap]
  611. yield from qubes_lvm_coro(cmd, self.log)
  612. elif not self.snap_on_start and not self.save_on_stop:
  613. cmd = ['remove', self.vid]
  614. yield from qubes_lvm_coro(cmd, self.log)
  615. finally:
  616. yield from reset_cache_coro()
  617. return self
  618. def verify(self):
  619. ''' Verifies the volume. '''
  620. if not self.save_on_stop and not self.snap_on_start:
  621. # volatile volumes don't need any files
  622. return True
  623. if self.source is not None:
  624. vid = self.source.path[len('/dev/'):]
  625. else:
  626. vid = self._vid_current
  627. try:
  628. vol_info = size_cache[vid]
  629. if vol_info['attr'][4] != 'a':
  630. raise qubes.storage.StoragePoolException(
  631. 'volume {} not active'.format(vid))
  632. except KeyError:
  633. raise qubes.storage.StoragePoolException(
  634. 'volume {} missing'.format(vid))
  635. return True
  636. def block_device(self):
  637. ''' Return :py:class:`qubes.storage.BlockDevice` for serialization in
  638. the libvirt XML template as <disk>.
  639. '''
  640. if self.snap_on_start or self.save_on_stop:
  641. return qubes.storage.BlockDevice(
  642. '/dev/' + self._vid_snap, self.name, self.script,
  643. self.rw, self.domain, self.devtype)
  644. return super(ThinVolume, self).block_device()
  645. @property
  646. def usage(self): # lvm thin usage always returns at least the same usage as
  647. # the parent
  648. refresh_cache()
  649. try:
  650. return qubes.storage.lvm.size_cache[self._vid_current]['usage']
  651. except KeyError:
  652. return 0
  653. def pool_exists(pool_id):
  654. ''' Return true if pool exists '''
  655. try:
  656. vol_info = size_cache[pool_id]
  657. return vol_info['attr'][0] == 't'
  658. except KeyError:
  659. return False
  660. def _get_lvm_cmdline(cmd):
  661. ''' Build command line for :program:`lvm` call.
  662. The purpose of this function is to keep all the detailed lvm options in
  663. one place.
  664. :param cmd: array of str, where cmd[0] is action and the rest are arguments
  665. :return array of str appropriate for subprocess.Popen
  666. '''
  667. action = cmd[0]
  668. if action == 'remove':
  669. lvm_cmd = ['lvremove', '-f', cmd[1]]
  670. elif action == 'clone':
  671. lvm_cmd = ['lvcreate', '-kn', '-ay', '-s', cmd[1], '-n', cmd[2]]
  672. elif action == 'create':
  673. lvm_cmd = ['lvcreate', '-T', cmd[1], '-kn', '-ay', '-n', cmd[2], '-V',
  674. str(cmd[3]) + 'B']
  675. elif action == 'extend':
  676. size = int(cmd[2]) / (1024 * 1024)
  677. lvm_cmd = ["lvextend", "-L%s" % size, cmd[1]]
  678. elif action == 'activate':
  679. lvm_cmd = ['lvchange', '-ay', cmd[1]]
  680. elif action == 'rename':
  681. lvm_cmd = ['lvrename', cmd[1], cmd[2]]
  682. else:
  683. raise NotImplementedError('unsupported action: ' + action)
  684. if lvm_is_very_old:
  685. # old lvm in trusty image used there does not support -k option
  686. lvm_cmd = [x for x in lvm_cmd if x != '-kn']
  687. if os.getuid() != 0:
  688. cmd = ['sudo', 'lvm'] + lvm_cmd
  689. else:
  690. cmd = ['lvm'] + lvm_cmd
  691. return cmd
  692. def _process_lvm_output(returncode, stdout, stderr, log):
  693. '''Process output of LVM, determine if the call was successful and
  694. possibly log warnings.'''
  695. # Filter out warning about intended over-provisioning.
  696. # Upstream discussion about missing option to silence it:
  697. # https://bugzilla.redhat.com/1347008
  698. err = '\n'.join(line for line in stderr.decode().splitlines()
  699. if 'exceeds the size of thin pool' not in line)
  700. if stdout:
  701. log.debug(stdout)
  702. if returncode == 0 and err:
  703. log.warning(err)
  704. elif returncode != 0:
  705. assert err, "Command exited unsuccessful, but printed nothing to stderr"
  706. err = err.replace('%', '%%')
  707. raise qubes.storage.StoragePoolException(err)
  708. return True
  709. def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')):
  710. ''' Call :program:`lvm` to execute an LVM operation '''
  711. # the only caller for this non-coroutine version is ThinVolume.export()
  712. cmd = _get_lvm_cmdline(cmd)
  713. environ = os.environ.copy()
  714. environ['LC_ALL'] = 'C.utf8'
  715. p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  716. close_fds=True, env=environ)
  717. out, err = p.communicate()
  718. return _process_lvm_output(p.returncode, out, err, log)
  719. @asyncio.coroutine
  720. def qubes_lvm_coro(cmd, log=logging.getLogger('qubes.storage.lvm')):
  721. ''' Call :program:`lvm` to execute an LVM operation
  722. Coroutine version of :py:func:`qubes_lvm`'''
  723. environ = os.environ.copy()
  724. environ['LC_ALL'] = 'C.utf8'
  725. if cmd[0] == "remove":
  726. pre_cmd = ['blkdiscard', '/dev/'+cmd[1]]
  727. p = yield from asyncio.create_subprocess_exec(*pre_cmd,
  728. stdout=subprocess.DEVNULL,
  729. stderr=subprocess.DEVNULL,
  730. close_fds=True, env=environ)
  731. _, _ = yield from p.communicate()
  732. cmd = _get_lvm_cmdline(cmd)
  733. p = yield from asyncio.create_subprocess_exec(*cmd,
  734. stdout=subprocess.PIPE,
  735. stderr=subprocess.PIPE,
  736. close_fds=True, env=environ)
  737. out, err = yield from p.communicate()
  738. return _process_lvm_output(p.returncode, out, err, log)
  739. def reset_cache():
  740. qubes.storage.lvm.size_cache = init_cache()
  741. qubes.storage.lvm.size_cache_time = time.monotonic()
  742. @asyncio.coroutine
  743. def reset_cache_coro():
  744. qubes.storage.lvm.size_cache = yield from init_cache_coro()
  745. qubes.storage.lvm.size_cache_time = time.monotonic()
  746. def refresh_cache():
  747. '''Reset size cache, if it's older than 30sec '''
  748. if size_cache_time+30 < time.monotonic():
  749. reset_cache()