backupcompatibility.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539
  1. #
  2. # The Qubes OS Project, http://www.qubes-os.org
  3. #
  4. # Copyright (C) 2014 Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
  5. #
  6. # This library is free software; you can redistribute it and/or
  7. # modify it under the terms of the GNU Lesser General Public
  8. # License as published by the Free Software Foundation; either
  9. # version 2.1 of the License, or (at your option) any later version.
  10. #
  11. # This library is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. # Lesser General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU Lesser General Public
  17. # License along with this library; if not, see <https://www.gnu.org/licenses/>.
  18. #
  19. from multiprocessing import Queue
  20. import asyncio
  21. import os
  22. import shutil
  23. import subprocess
  24. import unittest
  25. import sys
  26. import re
  27. import qubes.tests
  28. import qubes.tests.integ.backup
  29. QUBESXML_R2 = '''
  30. <QubesVmCollection updatevm="3" default_kernel="3.7.6-2" default_netvm="3" default_fw_netvm="2" default_template="1" clockvm="2">
  31. <QubesTemplateVm installed_by_rpm="True" kernel="3.7.6-2" uses_default_kernelopts="True" qid="1" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="fedora-20-x64.conf" label="black" template_qid="none" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{ 'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="fedora-20-x64" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/vm-templates/fedora-20-x64"/>
  32. <QubesNetVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="2" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="netvm.conf" label="red" template_qid="1" kernelopts="iommu=soft swiotlb=4096" memory="200" default_user="user" volatile_img="volatile.img" services="{'ntpd': False, 'meminfo-writer': False}" maxmem="1535" pcidevs="['02:00.0', '03:00.0']" name="netvm" netid="1" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/netvm"/>
  33. <QubesProxyVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="3" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="firewallvm.conf" label="green" template_qid="1" kernelopts="" memory="200" default_user="user" netvm_qid="2" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="firewallvm" netid="2" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/firewallvm"/>
  34. <QubesAppVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="4" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="True" conf_file="fedora-20-x64-dvm.conf" label="gray" template_qid="1" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{ 'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="fedora-20-x64-dvm" private_img="private.img" vcpus="1" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/fedora-20-x64-dvm"/>
  35. <QubesAppVm backup_content="True" backup_path="appvms/test-work" installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="5" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="test-work.conf" label="green" template_qid="1" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="test-work" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/test-work"/>
  36. <QubesAppVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="6" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="banking.conf" label="green" template_qid="1" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="banking" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/banking"/>
  37. <QubesAppVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="7" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="personal.conf" label="yellow" template_qid="1" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="personal" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/personal"/>
  38. <QubesAppVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="8" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="untrusted.conf" label="red" template_qid="1" kernelopts="" memory="400" default_user="user" netvm_qid="12" uses_default_netvm="False" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="untrusted" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/untrusted"/>
  39. <QubesTemplateVm backup_size="104857600" backup_content="True" backup_path="vm-templates/test-template-clone" installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="9" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="test-template-clone.conf" label="green" template_qid="none" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="test-template-clone" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/vm-templates/test-template-clone"/>
  40. <QubesAppVm backup_size="104857600" backup_content="True" backup_path="appvms/test-custom-template-appvm" installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="10" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="test-custom-template-appvm.conf" label="yellow" template_qid="9" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="test-custom-template-appvm" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/test-custom-template-appvm"/>
  41. <QubesAppVm backup_size="104857600" backup_content="True" backup_path="appvms/test-standalonevm" installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="11" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="test-standalonevm.conf" label="blue" template_qid="none" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="test-standalonevm" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/test-standalonevm"/>
  42. <QubesProxyVm backup_size="104857600" backup_content="True" backup_path="servicevms/test-testproxy" installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="12" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="test-testproxy.conf" label="red" template_qid="1" kernelopts="" memory="200" default_user="user" netvm_qid="3" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="test-testproxy" netid="3" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/test-testproxy"/>
  43. <QubesProxyVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="13" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="testproxy2.conf" label="red" template_qid="9" kernelopts="" memory="200" default_user="user" netvm_qid="2" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="testproxy2" netid="4" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/testproxy2"/>
  44. <QubesHVm backup_size="104857600" backup_content="True" backup_path="appvms/test-testhvm" installed_by_rpm="False" netvm_qid="none" qid="14" include_in_backups="True" timezone="localtime" qrexec_timeout="60" conf_file="test-testhvm.conf" label="purple" template_qid="none" internal="False" memory="512" uses_default_netvm="True" services="{'meminfo-writer': False}" default_user="user" pcidevs="[]" name="test-testhvm" qrexec_installed="False" private_img="private.img" drive="None" vcpus="2" root_img="root.img" guiagent_installed="False" debug="False" dir_path="/var/lib/qubes/appvms/test-testhvm"/>
  45. <QubesDisposableVm dispid="50" firewall_conf="firewall.xml" label="red" name="disp50" netvm_qid="2" qid="15" template_qid="1"/>
  46. </QubesVmCollection>
  47. '''
  48. MANGLED_SUBDIRS_R2 = {
  49. "test-work": "vm5",
  50. "test-template-clone": "vm9",
  51. "test-custom-template-appvm": "vm10",
  52. "test-standalonevm": "vm11",
  53. "test-testproxy": "vm12",
  54. "test-testhvm": "vm14",
  55. }
  56. APPTEMPLATE_R2B2 = '''
  57. [Desktop Entry]
  58. Name=%VMNAME%: {name}
  59. GenericName=%VMNAME%: {name}
  60. GenericName[ca]=%VMNAME%: Navegador web
  61. GenericName[cs]=%VMNAME%: Webový prohlížeč
  62. GenericName[es]=%VMNAME%: Navegador web
  63. GenericName[fa]=%VMNAME%: مرورر اینترنتی
  64. GenericName[fi]=%VMNAME%: WWW-selain
  65. GenericName[fr]=%VMNAME%: Navigateur Web
  66. GenericName[hu]=%VMNAME%: Webböngésző
  67. GenericName[it]=%VMNAME%: Browser Web
  68. GenericName[ja]=%VMNAME%: ウェブ・ブラウザ
  69. GenericName[ko]=%VMNAME%: 웹 브라우저
  70. GenericName[nb]=%VMNAME%: Nettleser
  71. GenericName[nl]=%VMNAME%: Webbrowser
  72. GenericName[nn]=%VMNAME%: Nettlesar
  73. GenericName[no]=%VMNAME%: Nettleser
  74. GenericName[pl]=%VMNAME%: Przeglądarka WWW
  75. GenericName[pt]=%VMNAME%: Navegador Web
  76. GenericName[pt_BR]=%VMNAME%: Navegador Web
  77. GenericName[sk]=%VMNAME%: Internetový prehliadač
  78. GenericName[sv]=%VMNAME%: Webbläsare
  79. Comment={comment}
  80. Comment[ca]=Navegueu per el web
  81. Comment[cs]=Prohlížení stránek World Wide Webu
  82. Comment[de]=Im Internet surfen
  83. Comment[es]=Navegue por la web
  84. Comment[fa]=صفحات شبه جهانی اینترنت را مرور نمایید
  85. Comment[fi]=Selaa Internetin WWW-sivuja
  86. Comment[fr]=Navigue sur Internet
  87. Comment[hu]=A világháló böngészése
  88. Comment[it]=Esplora il web
  89. Comment[ja]=ウェブを閲覧します
  90. Comment[ko]=웹을 돌아 다닙니다
  91. Comment[nb]=Surf på nettet
  92. Comment[nl]=Verken het internet
  93. Comment[nn]=Surf på nettet
  94. Comment[no]=Surf på nettet
  95. Comment[pl]=Przeglądanie stron WWW
  96. Comment[pt]=Navegue na Internet
  97. Comment[pt_BR]=Navegue na Internet
  98. Comment[sk]=Prehliadanie internetu
  99. Comment[sv]=Surfa på webben
  100. Exec=qvm-run -q --tray -a %VMNAME% '{command} %u'
  101. Categories=Network;WebBrowser;
  102. X-Qubes-VmName=%VMNAME%
  103. Icon=%VMDIR%/icon.png
  104. '''
  105. BACKUP_HEADER_R2 = '''version=3
  106. hmac-algorithm=SHA512
  107. crypto-algorithm=aes-256-cbc
  108. encrypted={encrypted}
  109. compressed={compressed}
  110. compression-filter=gzip
  111. '''
  112. class TC_00_BackupCompatibility(
  113. qubes.tests.integ.backup.BackupTestsMixin, qubes.tests.SystemTestCase):
  114. def tearDown(self):
  115. prefixes = ["test-", "disp-test-"]
  116. if 'disp-no-netvm' not in self.host_app.domains:
  117. prefixes.append('disp-no-netvm')
  118. self.remove_test_vms(prefix=prefixes)
  119. super(TC_00_BackupCompatibility, self).tearDown()
  120. def create_whitelisted_appmenus(self, filename):
  121. f = open(filename, "w")
  122. f.write("gnome-terminal.desktop\n")
  123. f.write("nautilus.desktop\n")
  124. f.write("firefox.desktop\n")
  125. f.write("mozilla-thunderbird.desktop\n")
  126. f.write("libreoffice-startcenter.desktop\n")
  127. f.close()
  128. def create_appmenus(self, dir, template, list):
  129. for name in list:
  130. f = open(os.path.join(dir, name + ".desktop"), "w")
  131. f.write(template.format(name=name, comment=name, command=name))
  132. f.close()
  133. def create_private_img(self, filename):
  134. self.create_sparse(filename, 2*2**30)
  135. subprocess.check_call(["/usr/sbin/mkfs.ext4", "-q", "-F", filename])
  136. def create_volatile_img(self, filename):
  137. self.create_sparse(filename, 11.5*2**30)
  138. # here used to be sfdisk call with "0,1024,S\n,10240,L\n" input,
  139. # but since sfdisk folks like to change command arguments in
  140. # incompatible way, have an partition table verbatim here
  141. ptable = (
  142. '\x00\x00\x00\x00\x00\x00\x00\x00\xab\x39\xd5\xd4\x00\x00\x20\x00'
  143. '\x00\x21\xaa\x82\x82\x28\x08\x00\x00\x00\x00\x00\x00\x20\xaa\x00'
  144. '\x82\x29\x15\x83\x9c\x79\x08\x00\x00\x20\x00\x00\x01\x40\x00\x00'
  145. '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
  146. '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xaa\x55'
  147. )
  148. with open(filename, 'r+') as f:
  149. f.seek(0x1b0)
  150. f.write(ptable)
  151. # TODO: mkswap
  152. def fullpath(self, name):
  153. return os.path.join(self.backupdir, name)
  154. def create_v1_files(self, r2b2=False):
  155. appmenus_list = [
  156. "firefox", "gnome-terminal", "evince", "evolution",
  157. "mozilla-thunderbird", "libreoffice-startcenter", "nautilus",
  158. "gedit", "gpk-update-viewer", "gpk-application"
  159. ]
  160. os.mkdir(self.fullpath("appvms"))
  161. os.mkdir(self.fullpath("servicevms"))
  162. os.mkdir(self.fullpath("vm-templates"))
  163. # normal AppVM
  164. os.mkdir(self.fullpath("appvms/test-work"))
  165. self.create_whitelisted_appmenus(self.fullpath(
  166. "appvms/test-work/whitelisted-appmenus.list"))
  167. os.symlink("/usr/share/qubes/icons/green.png",
  168. self.fullpath("appvms/test-work/icon.png"))
  169. self.create_private_img(self.fullpath("appvms/test-work/private.img"))
  170. # StandaloneVM
  171. os.mkdir(self.fullpath("appvms/test-standalonevm"))
  172. self.create_whitelisted_appmenus(self.fullpath(
  173. "appvms/test-standalonevm/whitelisted-appmenus.list"))
  174. os.symlink("/usr/share/qubes/icons/blue.png",
  175. self.fullpath("appvms/test-standalonevm/icon.png"))
  176. self.create_private_img(self.fullpath(
  177. "appvms/test-standalonevm/private.img"))
  178. self.create_sparse(
  179. self.fullpath("appvms/test-standalonevm/root.img"), 10*2**30)
  180. self.fill_image(self.fullpath("appvms/test-standalonevm/root.img"),
  181. 100*1024*1024, True)
  182. os.mkdir(self.fullpath("appvms/test-standalonevm/apps.templates"))
  183. self.create_appmenus(self.fullpath("appvms/test-standalonevm/apps"
  184. ".templates"),
  185. APPTEMPLATE_R2B2,
  186. appmenus_list)
  187. os.mkdir(self.fullpath("appvms/test-standalonevm/kernels"))
  188. for k_file in ["initramfs", "vmlinuz", "modules.img"]:
  189. self.fill_image(self.fullpath("appvms/test-standalonevm/kernels/"
  190. + k_file), 10*1024*1024)
  191. # VM based on custom template
  192. subprocess.check_call(
  193. ["/bin/cp", "-a", self.fullpath("appvms/test-work"),
  194. self.fullpath("appvms/test-custom-template-appvm")])
  195. # HVM
  196. if r2b2:
  197. subprocess.check_call(
  198. ["/bin/cp", "-a", self.fullpath("appvms/test-standalonevm"),
  199. self.fullpath("appvms/test-testhvm")])
  200. # ProxyVM
  201. os.mkdir(self.fullpath("servicevms/test-testproxy"))
  202. self.create_whitelisted_appmenus(self.fullpath(
  203. "servicevms/test-testproxy/whitelisted-appmenus.list"))
  204. self.create_private_img(
  205. self.fullpath("servicevms/test-testproxy/private.img"))
  206. # Custom template
  207. os.mkdir(self.fullpath("vm-templates/test-template-clone"))
  208. self.create_private_img(
  209. self.fullpath("vm-templates/test-template-clone/private.img"))
  210. self.create_sparse(self.fullpath(
  211. "vm-templates/test-template-clone/root-cow.img"), 10*2**30)
  212. self.create_sparse(self.fullpath(
  213. "vm-templates/test-template-clone/root.img"), 10*2**30)
  214. self.fill_image(self.fullpath(
  215. "vm-templates/test-template-clone/root.img"), 100*2**20, True)
  216. self.create_volatile_img(self.fullpath(
  217. "vm-templates/test-template-clone/volatile.img"))
  218. subprocess.check_call([
  219. "/bin/tar", "cS",
  220. "-f", self.fullpath(
  221. "vm-templates/test-template-clone/clean-volatile.img.tar"),
  222. "-C", self.fullpath("vm-templates/test-template-clone"),
  223. "volatile.img"])
  224. self.create_whitelisted_appmenus(self.fullpath(
  225. "vm-templates/test-template-clone/whitelisted-appmenus.list"))
  226. self.create_whitelisted_appmenus(self.fullpath(
  227. "vm-templates/test-template-clone/vm-whitelisted-appmenus.list"))
  228. if r2b2:
  229. self.create_whitelisted_appmenus(self.fullpath(
  230. "vm-templates/test-template-clone/netvm-whitelisted-appmenus"
  231. ".list"))
  232. os.symlink("/usr/share/qubes/icons/green.png",
  233. self.fullpath("vm-templates/test-template-clone/icon.png"))
  234. os.mkdir(
  235. self.fullpath("vm-templates/test-template-clone/apps.templates"))
  236. self.create_appmenus(
  237. self.fullpath("vm-templates/test-template-clone/apps.templates"),
  238. APPTEMPLATE_R2B2,
  239. appmenus_list)
  240. os.mkdir(self.fullpath("vm-templates/test-template-clone/apps"))
  241. self.create_appmenus(
  242. self.fullpath("vm-templates/test-template-clone/apps"),
  243. APPTEMPLATE_R2B2.replace("%VMNAME%", "test-template-clone")
  244. .replace("%VMDIR%", self.fullpath(
  245. "vm-templates/test-template-clone")),
  246. appmenus_list)
  247. def calculate_hmac(self, f_name, algorithm="sha512", password="qubes"):
  248. subprocess.check_call(["openssl", "dgst", "-"+algorithm, "-hmac",
  249. password],
  250. stdin=open(self.fullpath(f_name), "r"),
  251. stdout=open(self.fullpath(f_name+".hmac"), "w"))
  252. def append_backup_stream(self, f_name, stream, basedir=None):
  253. if not basedir:
  254. basedir = self.backupdir
  255. subprocess.check_call(["tar", "-cO", "--posix", "-C", basedir,
  256. f_name],
  257. stdout=stream)
  258. def handle_v3_file(self, f_name, subdir, stream, compressed=True,
  259. encrypted=True):
  260. # create inner archive
  261. tar_cmdline = ["tar", "-Pc", '--sparse',
  262. '-C', self.fullpath(os.path.dirname(f_name)),
  263. '--xform', 's:^%s:%s\\0:' % (
  264. os.path.basename(f_name),
  265. subdir),
  266. os.path.basename(f_name)
  267. ]
  268. if compressed:
  269. tar_cmdline.insert(-1, "--use-compress-program=%s" % "gzip")
  270. tar = subprocess.Popen(tar_cmdline, stdout=subprocess.PIPE)
  271. if encrypted:
  272. encryptor = subprocess.Popen(
  273. ["openssl", "enc", "-e", "-aes-256-cbc", "-pass", "pass:qubes"],
  274. stdin=tar.stdout,
  275. stdout=subprocess.PIPE)
  276. data = encryptor.stdout
  277. else:
  278. data = tar.stdout
  279. stage1_dir = self.fullpath(os.path.join("stage1", subdir))
  280. if not os.path.exists(stage1_dir):
  281. os.makedirs(stage1_dir)
  282. subprocess.check_call(["split", "--numeric-suffixes",
  283. "--suffix-length=3",
  284. "--bytes="+str(100*1024*1024), "-",
  285. os.path.join(stage1_dir,
  286. os.path.basename(f_name+"."))],
  287. stdin=data)
  288. for part in sorted(os.listdir(stage1_dir)):
  289. if not re.match(
  290. r"^{}.[0-9][0-9][0-9]$".format(os.path.basename(f_name)),
  291. part):
  292. continue
  293. part_with_dir = os.path.join(subdir, part)
  294. self.calculate_hmac(os.path.join("stage1", part_with_dir))
  295. self.append_backup_stream(part_with_dir, stream,
  296. basedir=self.fullpath("stage1"))
  297. self.append_backup_stream(part_with_dir+".hmac", stream,
  298. basedir=self.fullpath("stage1"))
  299. def create_v3_backup(self, encrypted=True, compressed=True):
  300. """
  301. Create "backup format 3" backup - used in R2 and R3.0
  302. :param encrypt: Should the backup be encrypted
  303. :return:
  304. """
  305. output = open(self.fullpath("backup.bin"), "w")
  306. f = open(self.fullpath("backup-header"), "w")
  307. f.write(BACKUP_HEADER_R2.format(
  308. encrypted=str(encrypted),
  309. compressed=str(compressed)
  310. ))
  311. f.close()
  312. self.calculate_hmac("backup-header")
  313. self.append_backup_stream("backup-header", output)
  314. self.append_backup_stream("backup-header.hmac", output)
  315. f = open(self.fullpath("qubes.xml"), "w")
  316. if encrypted:
  317. qubesxml = QUBESXML_R2
  318. for vmname, subdir in MANGLED_SUBDIRS_R2.items():
  319. qubesxml = re.sub(r"[a-z-]*/{}".format(vmname),
  320. subdir, qubesxml)
  321. f.write(qubesxml)
  322. else:
  323. f.write(QUBESXML_R2)
  324. f.close()
  325. self.handle_v3_file("qubes.xml", "", output, encrypted=encrypted,
  326. compressed=compressed)
  327. self.create_v1_files(r2b2=True)
  328. for vm_type in ["appvms", "servicevms"]:
  329. for vm_name in os.listdir(self.fullpath(vm_type)):
  330. vm_dir = os.path.join(vm_type, vm_name)
  331. for f_name in os.listdir(self.fullpath(vm_dir)):
  332. if encrypted:
  333. subdir = MANGLED_SUBDIRS_R2[vm_name]
  334. else:
  335. subdir = vm_dir
  336. self.handle_v3_file(
  337. os.path.join(vm_dir, f_name),
  338. subdir+'/', output, encrypted=encrypted)
  339. for vm_name in os.listdir(self.fullpath("vm-templates")):
  340. vm_dir = os.path.join("vm-templates", vm_name)
  341. if encrypted:
  342. subdir = MANGLED_SUBDIRS_R2[vm_name]
  343. else:
  344. subdir = vm_dir
  345. self.handle_v3_file(
  346. os.path.join(vm_dir, "."),
  347. subdir+'/', output, encrypted=encrypted)
  348. output.close()
  349. def assertRestored(self, name, **kwargs):
  350. # ignore automatically added features
  351. ignore_features = ['servicevm']
  352. with self.assertNotRaises((KeyError, qubes.exc.QubesException)):
  353. vm = self.app.domains[name]
  354. asyncio.get_event_loop().run_until_complete(vm.storage.verify())
  355. for prop, value in kwargs.items():
  356. if prop == 'klass':
  357. self.assertIsInstance(vm, value)
  358. elif prop == 'features':
  359. self.assertEqual(
  360. dict((k, v) for k, v in vm.features.items()
  361. if k not in ignore_features),
  362. value,
  363. 'VM {} - features mismatch'.format(vm.name))
  364. elif value is qubes.property.DEFAULT:
  365. self.assertTrue(vm.property_is_default(prop),
  366. 'VM {} - property {} not default'.format(vm.name, prop))
  367. else:
  368. actual_value = getattr(vm, prop)
  369. if isinstance(actual_value, qubes.vm.BaseVM):
  370. self.assertEqual(value, actual_value.name,
  371. 'VM {} - property {}'.format(vm.name, prop))
  372. elif isinstance(actual_value, qubes.Label):
  373. self.assertEqual(value, actual_value.name,
  374. 'VM {} - property {}'.format(vm.name, prop))
  375. else:
  376. self.assertEqual(value, actual_value,
  377. 'VM {} - property {}'.format(vm.name, prop))
  378. def test_210_r2(self):
  379. self.create_v3_backup(False)
  380. self.restore_backup(self.fullpath("backup.bin"), options={
  381. 'use-default-template': True,
  382. 'use-default-netvm': True,
  383. })
  384. common_props = {
  385. 'installed_by_rpm': False,
  386. 'kernel': qubes.property.DEFAULT,
  387. 'kernelopts': qubes.property.DEFAULT,
  388. 'qrexec_timeout': qubes.property.DEFAULT,
  389. 'netvm': qubes.property.DEFAULT,
  390. 'default_user': qubes.property.DEFAULT,
  391. 'include_in_backups': True,
  392. 'debug': False,
  393. 'maxmem': min(int(self.app.host.memory_total / 1024 / 2), 4000),
  394. 'memory': 400,
  395. 'features': {},
  396. }
  397. template_standalone_props = common_props.copy()
  398. template_standalone_props['features'] = {
  399. 'qrexec': '1',
  400. 'gui': '1',
  401. }
  402. self.assertRestored("test-template-clone",
  403. klass=qubes.vm.templatevm.TemplateVM,
  404. label='green',
  405. provides_network=False,
  406. **template_standalone_props)
  407. testproxy_props = common_props.copy()
  408. testproxy_props.update(
  409. label='red',
  410. provides_network=True,
  411. memory=200,
  412. template=self.app.default_template.name,
  413. )
  414. self.assertRestored("test-testproxy",
  415. klass=qubes.vm.appvm.AppVM,
  416. **testproxy_props)
  417. self.assertRestored("test-work",
  418. klass=qubes.vm.appvm.AppVM,
  419. template=self.app.default_template.name,
  420. label='green',
  421. **common_props)
  422. self.assertRestored("test-standalonevm",
  423. klass=qubes.vm.standalonevm.StandaloneVM,
  424. label='blue',
  425. **template_standalone_props)
  426. self.assertRestored("test-custom-template-appvm",
  427. klass=qubes.vm.appvm.AppVM,
  428. template='test-template-clone',
  429. label='yellow',
  430. **common_props)
  431. def test_220_r2_encrypted(self):
  432. self.create_v3_backup(True)
  433. self.restore_backup(self.fullpath("backup.bin"), options={
  434. 'use-default-template': True,
  435. 'use-default-netvm': True,
  436. })
  437. common_props = {
  438. 'installed_by_rpm': False,
  439. 'kernel': qubes.property.DEFAULT,
  440. 'kernelopts': qubes.property.DEFAULT,
  441. 'qrexec_timeout': qubes.property.DEFAULT,
  442. 'netvm': qubes.property.DEFAULT,
  443. 'default_user': qubes.property.DEFAULT,
  444. 'include_in_backups': True,
  445. 'debug': False,
  446. 'maxmem': min(int(self.app.host.memory_total / 1024 / 2), 4000),
  447. 'memory': 400,
  448. 'features': {},
  449. }
  450. template_standalone_props = common_props.copy()
  451. template_standalone_props['features'] = {
  452. 'qrexec': '1',
  453. 'gui': '1',
  454. }
  455. self.assertRestored("test-template-clone",
  456. klass=qubes.vm.templatevm.TemplateVM,
  457. label='green',
  458. provides_network=False,
  459. **template_standalone_props)
  460. testproxy_props = common_props.copy()
  461. testproxy_props.update(
  462. label='red',
  463. provides_network=True,
  464. memory=200,
  465. template=self.app.default_template.name,
  466. )
  467. self.assertRestored("test-testproxy",
  468. klass=qubes.vm.appvm.AppVM,
  469. **testproxy_props)
  470. self.assertRestored("test-work",
  471. klass=qubes.vm.appvm.AppVM,
  472. template=self.app.default_template.name,
  473. label='green',
  474. **common_props)
  475. self.assertRestored("test-standalonevm",
  476. klass=qubes.vm.standalonevm.StandaloneVM,
  477. label='blue',
  478. **template_standalone_props)
  479. self.assertRestored("test-custom-template-appvm",
  480. klass=qubes.vm.appvm.AppVM,
  481. template='test-template-clone',
  482. label='yellow',
  483. **common_props)
  484. class TC_01_BackupCompatibilityIntoLVM(TC_00_BackupCompatibility):
  485. def setUp(self):
  486. super(TC_01_BackupCompatibilityIntoLVM, self).setUp()
  487. self.init_lvm_pool()
  488. def restore_backup(self, source=None, appvm=None, options=None,
  489. **kwargs):
  490. if options is None:
  491. options = {}
  492. options['override_pool'] = self.pool.name
  493. super(TC_01_BackupCompatibilityIntoLVM, self).restore_backup(source,
  494. appvm, options, **kwargs)