__init__.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996
  1. #
  2. # The Qubes OS Project, https://www.qubes-os.org/
  3. #
  4. # Copyright (C) 2013-2015 Joanna Rutkowska <joanna@invisiblethingslab.com>
  5. # Copyright (C) 2013-2015 Marek Marczykowski-Górecki
  6. # <marmarek@invisiblethingslab.com>
  7. # Copyright (C) 2015 Wojtek Porczyk <woju@invisiblethingslab.com>
  8. #
  9. # This library is free software; you can redistribute it and/or
  10. # modify it under the terms of the GNU Lesser General Public
  11. # License as published by the Free Software Foundation; either
  12. # version 2.1 of the License, or (at your option) any later version.
  13. #
  14. # This library is distributed in the hope that it will be useful,
  15. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  17. # Lesser General Public License for more details.
  18. #
  19. # You should have received a copy of the GNU Lesser General Public
  20. # License along with this library; if not, see <https://www.gnu.org/licenses/>.
  21. #
  22. """ Qubes storage system"""
  23. import functools
  24. import inspect
  25. import os
  26. import os.path
  27. import string
  28. import subprocess
  29. import time
  30. from datetime import datetime
  31. import asyncio
  32. import lxml.etree
  33. import pkg_resources
  34. import qubes
  35. import qubes.exc
  36. import qubes.utils
  37. STORAGE_ENTRY_POINT = 'qubes.storage'
  38. class StoragePoolException(qubes.exc.QubesException):
  39. ''' A general storage exception '''
  40. class BlockDevice:
  41. ''' Represents a storage block device. '''
  42. # pylint: disable=too-few-public-methods
  43. def __init__(self, path, name, script=None, rw=True, domain=None,
  44. devtype='disk'):
  45. assert name, 'Missing device name'
  46. assert path, 'Missing device path'
  47. self.path = path
  48. self.name = name
  49. self.rw = rw
  50. self.script = script
  51. self.domain = domain
  52. self.devtype = devtype
  53. class Volume:
  54. ''' Encapsulates all data about a volume for serialization to qubes.xml and
  55. libvirt config.
  56. Keep in mind!
  57. volatile = not snap_on_start and not save_on_stop
  58. snapshot = snap_on_start and not save_on_stop
  59. origin = not snap_on_start and save_on_stop
  60. origin_snapshot = snap_on_start and save_on_stop
  61. '''
  62. devtype = 'disk'
  63. domain = None
  64. path = None
  65. script = None
  66. #: disk space used by this volume, can be smaller than :py:attr:`size`
  67. #: for sparse volumes
  68. usage = 0
  69. def __init__(self, name, pool, vid,
  70. revisions_to_keep=0, rw=False, save_on_stop=False, size=0,
  71. snap_on_start=False, source=None, **kwargs):
  72. ''' Initialize a volume.
  73. :param str name: The name of the volume inside owning domain
  74. :param Pool pool: The pool object
  75. :param str vid: Volume identifier needs to be unique in pool
  76. :param int revisions_to_keep: Amount of revisions to keep around
  77. :param bool rw: If true volume will be mounted read-write
  78. :param bool snap_on_start: Create a snapshot from source on
  79. start, instead of using volume own data
  80. :param bool save_on_stop: Write changes to the volume in
  81. vm.stop(), otherwise - discard
  82. :param Volume source: other volume in same pool to make snapshot
  83. from, required if *snap_on_start*=`True`
  84. :param str/int size: Size of the volume
  85. '''
  86. super().__init__(**kwargs)
  87. assert isinstance(pool, Pool)
  88. assert source is None or (isinstance(source, Volume)
  89. and source.pool == pool)
  90. if snap_on_start and source is None:
  91. msg = "snap_on_start specified on {!r} but no volume source set"
  92. msg = msg.format(name)
  93. raise StoragePoolException(msg)
  94. if not snap_on_start and source is not None:
  95. msg = "source specified on {!r} but no snap_on_start set"
  96. msg = msg.format(name)
  97. raise StoragePoolException(msg)
  98. #: Name of the volume in a domain it's attached to (like `root` or
  99. #: `private`).
  100. self.name = str(name)
  101. #: :py:class:`Pool` instance owning this volume
  102. self.pool = pool
  103. #: How many revisions of the volume to keep. Each revision is created
  104. # at :py:meth:`stop`, if :py:attr:`save_on_stop` is True
  105. self.revisions_to_keep = int(revisions_to_keep)
  106. #: Should this volume be writable by domain.
  107. self.rw = rw
  108. #: Should volume state be saved or discarded at :py:meth:`stop`
  109. self.save_on_stop = save_on_stop
  110. self._size = int(size)
  111. #: Should the volume state be initialized with a snapshot of
  112. #: same-named volume of domain's template.
  113. self.snap_on_start = snap_on_start
  114. #: source volume for :py:attr:`snap_on_start` volumes
  115. self.source = source
  116. #: Volume unique (inside given pool) identifier
  117. self.vid = vid
  118. #: Asynchronous lock for @Volume.locked decorator
  119. self._lock = asyncio.Lock()
  120. def __eq__(self, other):
  121. if isinstance(other, Volume):
  122. return other.pool == self.pool and other.vid == self.vid
  123. return NotImplemented
  124. def __hash__(self):
  125. return hash('%s:%s' % (self.pool, self.vid))
  126. def __neq__(self, other):
  127. return not self.__eq__(other)
  128. def __repr__(self):
  129. return '{!r}'.format(str(self.pool) + ':' + self.vid)
  130. def __str__(self):
  131. return str(self.vid)
  132. def __xml__(self):
  133. config = _sanitize_config(self.config)
  134. return lxml.etree.Element('volume', **config)
  135. @staticmethod
  136. def locked(method):
  137. '''Decorator running given Volume's coroutine under a lock.
  138. Needs to be added after wrapping with @asyncio.coroutine, for example:
  139. >>>@Volume.locked
  140. >>>@asyncio.coroutine
  141. >>>def start(self):
  142. >>> pass
  143. '''
  144. @asyncio.coroutine
  145. @functools.wraps(method)
  146. def wrapper(self, *args, **kwargs):
  147. with (yield from self._lock): # pylint: disable=protected-access
  148. return (yield from method(self, *args, **kwargs))
  149. return wrapper
  150. def create(self):
  151. ''' Create the given volume on disk.
  152. This method is called only once in the volume lifetime. Before
  153. calling this method, no data on disk should be touched (in
  154. context of this volume).
  155. This can be implemented as a coroutine.
  156. '''
  157. raise self._not_implemented("create")
  158. def remove(self):
  159. ''' Remove volume.
  160. This can be implemented as a coroutine.'''
  161. raise self._not_implemented("remove")
  162. def export(self):
  163. ''' Returns a path to read the volume data from.
  164. Reading from this path when domain owning this volume is
  165. running (i.e. when :py:meth:`is_dirty` is True) should return the
  166. data from before domain startup.
  167. Reading from the path returned by this method should return the
  168. volume data. If extracting volume data require something more
  169. than just reading from file (for example connecting to some other
  170. domain, or decompressing the data), the returned path may be a pipe.
  171. This can be implemented as a coroutine.
  172. '''
  173. raise self._not_implemented("export")
  174. def export_end(self, path):
  175. """ Cleanup after exporting data.
  176. This method is called after exporting the volume data (using
  177. :py:meth:`export`), when the *path* is not needed anymore.
  178. This can be implemented as a coroutine.
  179. :param path: path to cleanup, returned by :py:meth:`export`
  180. """
  181. # do nothing by default (optional method)
  182. def import_data(self, size):
  183. ''' Returns a path to overwrite volume data.
  184. This method is called after volume was already :py:meth:`create`-ed.
  185. Writing to this path should overwrite volume data. If importing
  186. volume data require something more than just writing to a file (
  187. for example connecting to some other domain, or converting data
  188. on the fly), the returned path may be a pipe.
  189. This can be implemented as a coroutine.
  190. :param int size: size of new data in bytes
  191. '''
  192. raise self._not_implemented("import_data")
  193. def import_data_end(self, success):
  194. ''' End the data import operation. This may be used by pool
  195. implementation to commit changes, cleanup temporary files etc.
  196. This method is called regardless the operation was successful or not.
  197. This can be implemented as a coroutine.
  198. :param success: True if data import was successful, otherwise False
  199. '''
  200. # by default do nothing
  201. def import_volume(self, src_volume):
  202. ''' Imports data from a different volume (possibly in a different
  203. pool.
  204. The volume needs to be create()d first.
  205. This can be implemented as a coroutine. '''
  206. # pylint: disable=unused-argument
  207. raise self._not_implemented("import_volume")
  208. def is_dirty(self):
  209. ''' Return `True` if volume was not properly shutdown and committed.
  210. This include the situation when domain owning the volume is still
  211. running.
  212. '''
  213. raise self._not_implemented("is_dirty")
  214. def is_outdated(self):
  215. ''' Returns `True` if this snapshot of a source volume (for
  216. `snap_on_start`=True) is outdated.
  217. '''
  218. raise self._not_implemented("is_outdated")
  219. def resize(self, size):
  220. ''' Expands volume, throws
  221. :py:class:`qubes.storage.StoragePoolException` if
  222. given size is less than current_size
  223. This can be implemented as a coroutine.
  224. :param int size: new size in bytes
  225. '''
  226. # pylint: disable=unused-argument
  227. raise self._not_implemented("resize")
  228. def revert(self, revision=None):
  229. ''' Revert volume to previous revision
  230. This can be implemented as a coroutine.
  231. :param revision: revision to revert volume to, see :py:attr:`revisions`
  232. '''
  233. # pylint: disable=unused-argument
  234. raise self._not_implemented("revert")
  235. def start(self):
  236. ''' Do what ever is needed on start.
  237. This include making a snapshot of template's volume if
  238. :py:attr:`snap_on_start` is set.
  239. This can be implemented as a coroutine.'''
  240. raise self._not_implemented("start")
  241. def stop(self):
  242. ''' Do what ever is needed on stop.
  243. This include committing data if :py:attr:`save_on_stop` is set.
  244. This can be implemented as a coroutine.'''
  245. raise self._not_implemented("stop")
  246. def verify(self):
  247. ''' Verifies the volume.
  248. This function is supposed to either return :py:obj:`True`, or raise
  249. an exception.
  250. This can be implemented as a coroutine.'''
  251. raise self._not_implemented("verify")
  252. def block_device(self):
  253. ''' Return :py:class:`BlockDevice` for serialization in
  254. the libvirt XML template as <disk>.
  255. '''
  256. return BlockDevice(self.path, self.name, self.script,
  257. self.rw, self.domain, self.devtype)
  258. @property
  259. def revisions(self):
  260. ''' Returns a dict containing revision identifiers and time of their
  261. creation '''
  262. msg = "{!s} has revisions not implemented".format(self.__class__)
  263. raise NotImplementedError(msg)
  264. @property
  265. def size(self):
  266. ''' Volume size in bytes '''
  267. return self._size
  268. @size.setter
  269. def size(self, size):
  270. # pylint: disable=attribute-defined-outside-init
  271. self._size = int(size)
  272. @property
  273. def config(self):
  274. ''' return config data for serialization to qubes.xml '''
  275. result = {
  276. 'name': self.name,
  277. 'pool': str(self.pool),
  278. 'vid': self.vid,
  279. 'revisions_to_keep': self.revisions_to_keep,
  280. 'rw': self.rw,
  281. 'save_on_stop': self.save_on_stop,
  282. 'snap_on_start': self.snap_on_start,
  283. }
  284. if self.size:
  285. result['size'] = self.size
  286. if self.source:
  287. result['source'] = str(self.source)
  288. return result
  289. def _not_implemented(self, method_name):
  290. ''' Helper for emitting helpful `NotImplementedError` exceptions '''
  291. msg = "Volume {!s} has {!s}() not implemented"
  292. msg = msg.format(str(self.__class__.__name__), method_name)
  293. return NotImplementedError(msg)
  294. class Storage:
  295. ''' Class for handling VM virtual disks.
  296. This is base class for all other implementations, mostly with Xen on Linux
  297. in mind.
  298. '''
  299. AVAILABLE_FRONTENDS = {'xvd' + c for c in string.ascii_lowercase}
  300. def __init__(self, vm):
  301. #: Domain for which we manage storage
  302. self.vm = vm
  303. self.log = self.vm.log
  304. #: Additional drive (currently used only by HVM)
  305. self.drive = None
  306. if hasattr(vm, 'volume_config'):
  307. for name, conf in self.vm.volume_config.items():
  308. self.init_volume(name, conf)
  309. def _update_volume_config_source(self, name, volume_config):
  310. '''Retrieve 'source' volume from VM's template'''
  311. template = getattr(self.vm, 'template', None)
  312. # recursively lookup source volume - templates may be
  313. # chained (TemplateVM -> AppVM -> DispVM, where the
  314. # actual source should be used from TemplateVM)
  315. while template:
  316. source = template.volumes[name]
  317. volume_config['source'] = source
  318. volume_config['pool'] = source.pool
  319. volume_config['size'] = source.size
  320. if source.source is not None:
  321. template = getattr(template, 'template', None)
  322. else:
  323. break
  324. def init_volume(self, name, volume_config):
  325. ''' Initialize Volume instance attached to this domain '''
  326. if 'name' not in volume_config:
  327. volume_config['name'] = name
  328. if 'source' in volume_config:
  329. # we have no control over VM load order,
  330. # so initialize storage recursively if needed
  331. template = getattr(self.vm, 'template', None)
  332. if template and template.storage is None:
  333. template.storage = Storage(template)
  334. if volume_config['source'] is None:
  335. self._update_volume_config_source(name, volume_config)
  336. else:
  337. # if source is already specified, pool needs to be too
  338. pool = self.vm.app.get_pool(volume_config['pool'])
  339. volume_config['source'] = pool.volumes[volume_config['source']]
  340. # if pool still unknown, load default
  341. if 'pool' not in volume_config:
  342. volume_config['pool'] = \
  343. getattr(self.vm.app, 'default_pool_' + name)
  344. pool = self.vm.app.get_pool(volume_config['pool'])
  345. if 'internal' in volume_config:
  346. # migrate old config
  347. del volume_config['internal']
  348. volume = pool.init_volume(self.vm, volume_config.copy())
  349. self.vm.volumes[name] = volume
  350. return volume
  351. def attach(self, volume, rw=False):
  352. ''' Attach a volume to the domain '''
  353. assert self.vm.is_running()
  354. if self._is_already_attached(volume):
  355. self.vm.log.info("{!r} already attached".format(volume))
  356. return
  357. try:
  358. frontend = self.unused_frontend()
  359. except IndexError:
  360. raise StoragePoolException("No unused frontend found")
  361. disk = lxml.etree.Element("disk")
  362. disk.set('type', 'block')
  363. disk.set('device', 'disk')
  364. lxml.etree.SubElement(disk, 'driver').set('name', 'phy')
  365. lxml.etree.SubElement(disk, 'source').set('dev', '/dev/%s' % volume.vid)
  366. lxml.etree.SubElement(disk, 'target').set('dev', frontend)
  367. if not rw:
  368. lxml.etree.SubElement(disk, 'readonly')
  369. if volume.domain is not None:
  370. lxml.etree.SubElement(disk, 'backenddomain').set(
  371. 'name', volume.domain.name)
  372. xml_string = lxml.etree.tostring(disk, encoding='utf-8')
  373. self.vm.libvirt_domain.attachDevice(xml_string)
  374. # trigger watches to update device status
  375. # FIXME: this should be removed once libvirt will report such
  376. # events itself
  377. # self.vm.untrusted_qdb.write('/qubes-block-devices', '')
  378. # ← do we need this?
  379. def _is_already_attached(self, volume):
  380. ''' Checks if the given volume is already attached '''
  381. parsed_xml = lxml.etree.fromstring(self.vm.libvirt_domain.XMLDesc())
  382. disk_sources = parsed_xml.xpath("//domain/devices/disk/source")
  383. for source in disk_sources:
  384. if source.get('dev') == '/dev/%s' % volume.vid:
  385. return True
  386. return False
  387. def detach(self, volume):
  388. ''' Detach a volume from domain '''
  389. parsed_xml = lxml.etree.fromstring(self.vm.libvirt_domain.XMLDesc())
  390. disks = parsed_xml.xpath("//domain/devices/disk")
  391. for disk in disks:
  392. source = disk.xpath('source')[0]
  393. if source.get('dev') == '/dev/%s' % volume.vid:
  394. disk_xml = lxml.etree.tostring(disk, encoding='utf-8')
  395. self.vm.libvirt_domain.detachDevice(disk_xml)
  396. return
  397. raise StoragePoolException('Volume {!r} is not attached'.format(volume))
  398. @property
  399. def kernels_dir(self):
  400. '''Directory where kernel resides.
  401. If :py:attr:`self.vm.kernel` is :py:obj:`None`, the this points inside
  402. :py:attr:`self.vm.dir_path`
  403. '''
  404. if not self.vm.kernel:
  405. return None
  406. if 'kernel' in self.vm.volumes:
  407. return self.vm.volumes['kernel'].kernels_dir
  408. return os.path.join(
  409. qubes.config.qubes_base_dir,
  410. qubes.config.system_path['qubes_kernels_base_dir'],
  411. self.vm.kernel)
  412. def get_disk_utilization(self):
  413. ''' Returns summed up disk utilization for all domain volumes '''
  414. result = 0
  415. for volume in self.vm.volumes.values():
  416. result += volume.usage
  417. return result
  418. @asyncio.coroutine
  419. def resize(self, volume, size):
  420. ''' Resizes volume a read-writable volume '''
  421. if isinstance(volume, str):
  422. volume = self.vm.volumes[volume]
  423. yield from qubes.utils.coro_maybe(volume.resize(size))
  424. if self.vm.is_running():
  425. try:
  426. yield from self.vm.run_service_for_stdio('qubes.ResizeDisk',
  427. input=volume.name.encode(),
  428. user='root')
  429. except subprocess.CalledProcessError as e:
  430. service_error = e.stderr.decode('ascii', errors='ignore')
  431. service_error = service_error.replace('%', '')
  432. raise StoragePoolException(
  433. 'Online resize of volume {} failed (you need to resize '
  434. 'filesystem manually): {}'.format(volume, service_error))
  435. @asyncio.coroutine
  436. def create(self):
  437. ''' Creates volumes on disk '''
  438. old_umask = os.umask(0o002)
  439. yield from qubes.utils.void_coros_maybe(
  440. vol.create() for vol in self.vm.volumes.values())
  441. os.umask(old_umask)
  442. @asyncio.coroutine
  443. def clone_volume(self, src_vm, name):
  444. ''' Clone single volume from the specified vm
  445. :param QubesVM src_vm: source VM
  446. :param str name: name of volume to clone ('root', 'private' etc)
  447. :return cloned volume object
  448. '''
  449. config = self.vm.volume_config[name]
  450. dst_pool = self.vm.app.get_pool(config['pool'])
  451. dst = dst_pool.init_volume(self.vm, config)
  452. src_volume = src_vm.volumes[name]
  453. msg = "Importing volume {!s} from vm {!s}"
  454. self.vm.log.info(msg.format(src_volume.name, src_vm.name))
  455. yield from qubes.utils.coro_maybe(dst.create())
  456. yield from qubes.utils.coro_maybe(dst.import_volume(src_volume))
  457. self.vm.volumes[name] = dst
  458. return self.vm.volumes[name]
  459. @asyncio.coroutine
  460. def clone(self, src_vm):
  461. ''' Clone volumes from the specified vm '''
  462. self.vm.volumes = {}
  463. with VmCreationManager(self.vm):
  464. yield from qubes.utils.void_coros_maybe(
  465. self.clone_volume(src_vm, vol_name)
  466. for vol_name in self.vm.volume_config.keys())
  467. @property
  468. def outdated_volumes(self):
  469. ''' Returns a list of outdated volumes '''
  470. result = []
  471. if self.vm.is_halted():
  472. return result
  473. volumes = self.vm.volumes
  474. for volume in volumes.values():
  475. if volume.is_outdated():
  476. result += [volume]
  477. return result
  478. @asyncio.coroutine
  479. def verify(self):
  480. '''Verify that the storage is sane.
  481. On success, returns normally. On failure, raises exception.
  482. '''
  483. if not os.path.exists(self.vm.dir_path):
  484. raise qubes.exc.QubesVMError(
  485. self.vm,
  486. 'VM directory does not exist: {}'.format(self.vm.dir_path))
  487. yield from qubes.utils.void_coros_maybe(
  488. vol.verify() for vol in self.vm.volumes.values())
  489. self.vm.fire_event('domain-verify-files')
  490. return True
  491. @asyncio.coroutine
  492. def remove(self):
  493. ''' Remove all the volumes.
  494. Errors on removal are catched and logged.
  495. '''
  496. results = []
  497. for vol in self.vm.volumes.values():
  498. self.log.info('Removing volume %s: %s' % (vol.name, vol.vid))
  499. try:
  500. results.append(vol.remove())
  501. except (IOError, OSError) as e:
  502. self.vm.log.exception("Failed to remove volume %s", vol.name, e)
  503. try:
  504. yield from qubes.utils.void_coros_maybe(results)
  505. except (IOError, OSError) as e:
  506. self.vm.log.exception("Failed to remove some volume", e)
  507. @asyncio.coroutine
  508. def start(self):
  509. ''' Execute the start method on each volume '''
  510. yield from qubes.utils.void_coros_maybe(
  511. vol.start() for vol in self.vm.volumes.values())
  512. @asyncio.coroutine
  513. def stop(self):
  514. ''' Execute the stop method on each volume '''
  515. yield from qubes.utils.void_coros_maybe(
  516. vol.stop() for vol in self.vm.volumes.values())
  517. def unused_frontend(self):
  518. ''' Find an unused device name '''
  519. unused_frontends = self.AVAILABLE_FRONTENDS.difference(
  520. self.used_frontends)
  521. return sorted(unused_frontends)[0]
  522. @property
  523. def used_frontends(self):
  524. ''' Used device names '''
  525. xml = self.vm.libvirt_domain.XMLDesc()
  526. parsed_xml = lxml.etree.fromstring(xml)
  527. return {target.get('dev', None)
  528. for target in parsed_xml.xpath(
  529. "//domain/devices/disk/target")}
  530. @asyncio.coroutine
  531. def export(self, volume):
  532. ''' Helper function to export volume (pool.export(volume))'''
  533. assert isinstance(volume, (Volume, str)), \
  534. "You need to pass a Volume or pool name as str"
  535. if not isinstance(volume, Volume):
  536. volume = self.vm.volumes[volume]
  537. return (yield from qubes.utils.coro_maybe(volume.export()))
  538. @asyncio.coroutine
  539. def export_end(self, volume, export_path):
  540. """ Cleanup after exporting data from the volume
  541. :param volume: volume that was exported
  542. :param export_path: path returned by the export() call
  543. """
  544. assert isinstance(volume, (Volume, str)), \
  545. "You need to pass a Volume or pool name as str"
  546. if not isinstance(volume, Volume):
  547. volume = self.vm.volumes[volume]
  548. yield from qubes.utils.coro_maybe(volume.export_end(export_path))
  549. @asyncio.coroutine
  550. def import_data(self, volume, size):
  551. '''
  552. Helper function to import volume data (pool.import_data(volume)).
  553. :size: new size in bytes, or None if using old size
  554. '''
  555. assert isinstance(volume, (Volume, str)), \
  556. "You need to pass a Volume or pool name as str"
  557. if isinstance(volume, str):
  558. volume = self.vm.volumes[volume]
  559. if size is None:
  560. size = volume.size
  561. ret = volume.import_data(size)
  562. return (yield from qubes.utils.coro_maybe(ret))
  563. @asyncio.coroutine
  564. def import_data_end(self, volume, success):
  565. ''' Helper function to finish/cleanup data import
  566. (pool.import_data_end( volume))'''
  567. assert isinstance(volume, (Volume, str)), \
  568. "You need to pass a Volume or pool name as str"
  569. if isinstance(volume, Volume):
  570. ret = volume.import_data_end(success=success)
  571. else:
  572. ret = self.vm.volumes[volume].import_data_end(success=success)
  573. return (yield from qubes.utils.coro_maybe(ret))
  574. class VolumesCollection:
  575. '''Convenient collection wrapper for pool.get_volume and
  576. pool.list_volumes
  577. '''
  578. def __init__(self, pool):
  579. self._pool = pool
  580. def __getitem__(self, item):
  581. ''' Get a single volume with given Volume ID.
  582. You can also a Volume instance to get the same Volume or KeyError if
  583. Volume no longer exists.
  584. :param item: a Volume ID (str) or a Volume instance
  585. '''
  586. if isinstance(item, Volume):
  587. if item.pool == self._pool:
  588. return self[item.vid]
  589. raise KeyError(item)
  590. try:
  591. return self._pool.get_volume(item)
  592. except NotImplementedError:
  593. for vol in self:
  594. if vol.vid == item:
  595. return vol
  596. # if list_volumes is not implemented too, it will raise
  597. # NotImplementedError again earlier
  598. raise KeyError(item)
  599. def __iter__(self):
  600. ''' Get iterator over pool's volumes '''
  601. return iter(self._pool.list_volumes())
  602. def __contains__(self, item):
  603. ''' Check if given volume (either Volume ID or Volume instance) is
  604. present in the pool
  605. '''
  606. try:
  607. return self[item] is not None
  608. except KeyError:
  609. return False
  610. def keys(self):
  611. ''' Return list of volume IDs '''
  612. return [vol.vid for vol in self]
  613. def values(self):
  614. ''' Return list of Volumes'''
  615. return list(self)
  616. class Pool:
  617. ''' A Pool is used to manage different kind of volumes (File
  618. based/LVM/Btrfs/...).
  619. 3rd Parties providing own storage implementations will need to extend
  620. this class.
  621. ''' # pylint: disable=unused-argument
  622. private_img_size = qubes.config.defaults['private_img_size']
  623. root_img_size = qubes.config.defaults['root_img_size']
  624. def __init__(self, *, name, revisions_to_keep=1):
  625. self._volumes_collection = VolumesCollection(self)
  626. self.name = name
  627. self.revisions_to_keep = revisions_to_keep
  628. def __eq__(self, other):
  629. if isinstance(other, Pool):
  630. return self.name == other.name
  631. if isinstance(other, str):
  632. return self.name == other
  633. return NotImplemented
  634. def __neq__(self, other):
  635. return not self.__eq__(other)
  636. def __str__(self):
  637. return self.name
  638. def __hash__(self):
  639. return hash(self.name)
  640. def __xml__(self):
  641. config = _sanitize_config(self.config)
  642. return lxml.etree.Element('pool', **config)
  643. @property
  644. def config(self):
  645. ''' Returns the pool config to be written to qubes.xml '''
  646. raise self._not_implemented("config")
  647. def destroy(self):
  648. ''' Called when removing the pool. Use this for implementation specific
  649. clean up.
  650. This can be implemented as a coroutine.
  651. '''
  652. raise self._not_implemented("destroy")
  653. def init_volume(self, vm, volume_config):
  654. ''' Initialize a :py:class:`qubes.storage.Volume` from `volume_config`.
  655. '''
  656. raise self._not_implemented("init_volume")
  657. def setup(self):
  658. ''' Called when adding a pool to the system. Use this for implementation
  659. specific set up.
  660. This can be implemented as a coroutine.
  661. '''
  662. raise self._not_implemented("setup")
  663. @property
  664. def volumes(self):
  665. ''' Return a collection of volumes managed by this pool '''
  666. return self._volumes_collection
  667. def list_volumes(self):
  668. ''' Return a list of volumes managed by this pool '''
  669. raise self._not_implemented("list_volumes")
  670. def get_volume(self, vid):
  671. ''' Return a volume with *vid* from this pool
  672. :raise KeyError: if no volume is found
  673. '''
  674. raise self._not_implemented("get_volume")
  675. def included_in(self, app):
  676. ''' Check if this pool is physically included in another one
  677. This works on best-effort basis, because one pool driver may not know
  678. all the other drivers.
  679. :param app: Qubes() object to lookup other pools in
  680. :returns pool or None
  681. '''
  682. @property
  683. def size(self):
  684. ''' Storage pool size in bytes, or None if unknown '''
  685. @property
  686. def usage(self):
  687. ''' Space used in the pool in bytes, or None if unknown '''
  688. @property
  689. def usage_details(self):
  690. """Detailed information about pool usage as a dictionary
  691. Contains data_usage for usage in bytes and data_size for pool
  692. size; other implementations may add more implementation-specific
  693. detail"""
  694. result = {}
  695. if self.usage is not None:
  696. result['data_usage'] = self.usage
  697. if self.size is not None:
  698. result['data_size'] = self.size
  699. return result
  700. def _not_implemented(self, method_name):
  701. ''' Helper for emitting helpful `NotImplementedError` exceptions '''
  702. msg = "Pool driver {!s} has {!s}() not implemented"
  703. msg = msg.format(str(self.__class__.__name__), method_name)
  704. return NotImplementedError(msg)
  705. def _sanitize_config(config):
  706. ''' Helper function to convert types to appropriate strings
  707. ''' # FIXME: find another solution for serializing basic types
  708. result = {}
  709. for key, value in config.items():
  710. if isinstance(value, bool):
  711. if value:
  712. result[key] = 'True'
  713. else:
  714. result[key] = str(value)
  715. return result
  716. def pool_drivers():
  717. """ Return a list of EntryPoints names """
  718. return [ep.name
  719. for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)]
  720. def driver_parameters(name):
  721. ''' Get __init__ parameters from a driver with out `self` & `name`. '''
  722. init_function = qubes.utils.get_entry_point_one(
  723. qubes.storage.STORAGE_ENTRY_POINT, name).__init__
  724. signature = inspect.signature(init_function)
  725. params = signature.parameters.keys()
  726. ignored_params = ['self', 'name', 'kwargs']
  727. return [p for p in params if p not in ignored_params]
  728. def isodate(seconds):
  729. ''' Helper method which returns an iso date '''
  730. return datetime.utcfromtimestamp(seconds).isoformat("T")
  731. def search_pool_containing_dir(pools, dir_path):
  732. ''' Helper function looking for a pool containing given directory.
  733. This is useful for implementing Pool.included_in method
  734. '''
  735. real_dir_path = os.path.realpath(dir_path)
  736. # prefer filesystem pools
  737. for pool in pools:
  738. if hasattr(pool, 'dir_path'):
  739. pool_real_dir_path = os.path.realpath(pool.dir_path)
  740. if os.path.commonpath([pool_real_dir_path, real_dir_path]) == \
  741. pool_real_dir_path:
  742. return pool
  743. # then look for lvm
  744. for pool in pools:
  745. if hasattr(pool, 'thin_pool') and hasattr(pool, 'volume_group'):
  746. if (pool.volume_group, pool.thin_pool) == \
  747. DirectoryThinPool.thin_pool(real_dir_path):
  748. return pool
  749. return None
  750. class VmCreationManager:
  751. ''' A `ContextManager` which cleans up if volume creation fails.
  752. ''' # pylint: disable=too-few-public-methods
  753. def __init__(self, vm):
  754. self.vm = vm
  755. def __enter__(self):
  756. pass
  757. def __exit__(self, type, value, tb): # pylint: disable=redefined-builtin
  758. if type is not None and value is not None and tb is not None:
  759. for volume in self.vm.volumes.values():
  760. try:
  761. volume.remove()
  762. except Exception: # pylint: disable=broad-except
  763. pass
  764. os.rmdir(self.vm.dir_path)
  765. # pylint: disable=too-few-public-methods
  766. class DirectoryThinPool:
  767. '''The thin pool containing the device of given filesystem'''
  768. _thin_pool = {}
  769. @classmethod
  770. def _init(cls, dir_path):
  771. '''Find out the thin pool containing given filesystem'''
  772. if dir_path not in cls._thin_pool:
  773. cls._thin_pool[dir_path] = None, None
  774. try:
  775. fs_stat = os.stat(dir_path)
  776. fs_major = (fs_stat.st_dev & 0xff00) >> 8
  777. fs_minor = fs_stat.st_dev & 0xff
  778. sudo = []
  779. if os.getuid():
  780. sudo = ['sudo']
  781. root_table = subprocess.check_output(sudo + ["dmsetup",
  782. "-j", str(fs_major), "-m", str(fs_minor),
  783. "table"], stderr=subprocess.DEVNULL)
  784. _start, _sectors, target_type, target_args = \
  785. root_table.decode().split(" ", 3)
  786. if target_type == "thin":
  787. thin_pool_devnum, _thin_pool_id = target_args.split(" ")
  788. with open("/sys/dev/block/{}/dm/name"
  789. .format(thin_pool_devnum), "r") as thin_pool_tpool_f:
  790. thin_pool_tpool = thin_pool_tpool_f.read().rstrip('\n')
  791. if thin_pool_tpool.endswith("-tpool"):
  792. # LVM replaces '-' by '--' if name contains
  793. # a hyphen
  794. thin_pool_tpool = thin_pool_tpool.replace('--', '=')
  795. volume_group, thin_pool, _tpool = \
  796. thin_pool_tpool.rsplit("-", 2)
  797. volume_group = volume_group.replace('=', '-')
  798. thin_pool = thin_pool.replace('=', '-')
  799. cls._thin_pool[dir_path] = volume_group, thin_pool
  800. except: # pylint: disable=bare-except
  801. pass
  802. @classmethod
  803. def thin_pool(cls, dir_path):
  804. '''Thin tuple (volume group, pool name) containing given filesystem'''
  805. cls._init(dir_path)
  806. return cls._thin_pool[dir_path]