backupcompatibility.py 82 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989
  1. # -*- encoding: utf8 -*-
  2. #
  3. # The Qubes OS Project, http://www.qubes-os.org
  4. #
  5. # Copyright (C) 2014 Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
  6. #
  7. # This program is free software; you can redistribute it and/or
  8. # modify it under the terms of the GNU Lesser General Public License
  9. # as published by the Free Software Foundation; either version 2.1
  10. # of the License, or (at your option) any later version.
  11. #
  12. # This program is distributed in the hope that it will be useful,
  13. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. # GNU Lesser General Public License for more details.
  16. #
  17. # You should have received a copy of the GNU Lesser General Public License
  18. # along with this program; if not, write to the Free Software
  19. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20. #
  21. import functools
  22. import shutil
  23. import tempfile
  24. import unittest
  25. from distutils import spawn
  26. from multiprocessing import Queue
  27. import os
  28. import subprocess
  29. import logging
  30. import time
  31. import contextlib
  32. try:
  33. import unittest.mock as mock
  34. except ImportError:
  35. import mock
  36. import re
  37. import multiprocessing
  38. import pkg_resources
  39. import sys
  40. import qubesadmin.backup.core2
  41. import qubesadmin.backup.core3
  42. import qubesadmin.exc
  43. import qubesadmin.firewall
  44. import qubesadmin.storage
  45. import qubesadmin.tests
  46. import qubesadmin.tests.backup
  47. MANGLED_SUBDIRS_R2 = {
  48. "test-work": "vm5",
  49. "test-template-clone": "vm9",
  50. "test-custom-template-appvm": "vm10",
  51. "test-standalonevm": "vm11",
  52. "test-testproxy": "vm12",
  53. "test-testhvm": "vm14",
  54. "test-net": "vm16",
  55. }
  56. MANGLED_SUBDIRS_R4 = {
  57. "test-work": "vm3",
  58. "test-fedora-25-clone": "vm7",
  59. "test-custom-template-appvm": "vm31",
  60. "test-standalonevm": "vm4",
  61. "test-proxy": "vm30",
  62. "test-hvm": "vm9",
  63. "test-net": "vm6",
  64. "test-d8test": "vm20",
  65. }
  66. APPTEMPLATE_R2B2 = '''
  67. [Desktop Entry]
  68. Name=%VMNAME%: {name}
  69. GenericName=%VMNAME%: {name}
  70. GenericName[ca]=%VMNAME%: Navegador web
  71. GenericName[cs]=%VMNAME%: Webový prohlížeč
  72. GenericName[es]=%VMNAME%: Navegador web
  73. GenericName[fa]=%VMNAME%: مرورر اینترنتی
  74. GenericName[fi]=%VMNAME%: WWW-selain
  75. GenericName[fr]=%VMNAME%: Navigateur Web
  76. GenericName[hu]=%VMNAME%: Webböngésző
  77. GenericName[it]=%VMNAME%: Browser Web
  78. GenericName[ja]=%VMNAME%: ウェブ・ブラウザ
  79. GenericName[ko]=%VMNAME%: 웹 브라우저
  80. GenericName[nb]=%VMNAME%: Nettleser
  81. GenericName[nl]=%VMNAME%: Webbrowser
  82. GenericName[nn]=%VMNAME%: Nettlesar
  83. GenericName[no]=%VMNAME%: Nettleser
  84. GenericName[pl]=%VMNAME%: Przeglądarka WWW
  85. GenericName[pt]=%VMNAME%: Navegador Web
  86. GenericName[pt_BR]=%VMNAME%: Navegador Web
  87. GenericName[sk]=%VMNAME%: Internetový prehliadač
  88. GenericName[sv]=%VMNAME%: Webbläsare
  89. Comment={comment}
  90. Comment[ca]=Navegueu per el web
  91. Comment[cs]=Prohlížení stránek World Wide Webu
  92. Comment[de]=Im Internet surfen
  93. Comment[es]=Navegue por la web
  94. Comment[fa]=صفحات شبه جهانی اینترنت را مرور نمایید
  95. Comment[fi]=Selaa Internetin WWW-sivuja
  96. Comment[fr]=Navigue sur Internet
  97. Comment[hu]=A világháló böngészése
  98. Comment[it]=Esplora il web
  99. Comment[ja]=ウェブを閲覧します
  100. Comment[ko]=웹을 돌아 다닙니다
  101. Comment[nb]=Surf på nettet
  102. Comment[nl]=Verken het internet
  103. Comment[nn]=Surf på nettet
  104. Comment[no]=Surf på nettet
  105. Comment[pl]=Przeglądanie stron WWW
  106. Comment[pt]=Navegue na Internet
  107. Comment[pt_BR]=Navegue na Internet
  108. Comment[sk]=Prehliadanie internetu
  109. Comment[sv]=Surfa på webben
  110. Exec=qvm-run -q --tray -a %VMNAME% '{command} %u'
  111. Categories=Network;WebBrowser;
  112. X-Qubes-VmName=%VMNAME%
  113. Icon=%VMDIR%/icon.png
  114. '''
  115. QUBESXML_R1 = '''<?xml version='1.0' encoding='UTF-8'?>
  116. <QubesVmCollection clockvm="2" default_fw_netvm="2" default_kernel="3.2.7-10" default_netvm="3" default_template="1" updatevm="3"><QubesTemplateVm conf_file="fedora-17-x64.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/vm-templates/fedora-17-x64" include_in_backups="True" installed_by_rpm="True" internal="False" kernel="3.2.7-10" kernelopts="" label="gray" maxmem="4063" memory="400" name="fedora-17-x64" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="1" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="none" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesNetVm conf_file="netvm.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/servicevms/netvm" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="iommu=soft swiotlb=2048" label="red" maxmem="4063" memory="200" name="netvm" netid="1" pcidevs="[&apos;00:19.0&apos;, &apos;03:00.0&apos;]" private_img="private.img" qid="2" root_img="root.img" services="{&apos;ntpd&apos;: False, &apos;meminfo-writer&apos;: False}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" vcpus="2" volatile_img="volatile.img" /><QubesProxyVm conf_file="firewallvm.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/servicevms/firewallvm" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="green" maxmem="4063" memory="200" name="firewallvm" netid="2" netvm_qid="2" pcidevs="[]" private_img="private.img" qid="3" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="fedora-17-x64-dvm.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/fedora-17-x64-dvm" include_in_backups="True" installed_by_rpm="False" internal="True" kernel="3.2.7-10" kernelopts="" label="gray" maxmem="4063" memory="400" name="fedora-17-x64-dvm" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="4" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="1" volatile_img="volatile.img" /><QubesAppVm conf_file="test-work.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/test-work" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="green" maxmem="4063" memory="400" name="test-work" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="5" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="personal.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/personal" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="yellow" maxmem="4063" memory="400" name="personal" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="6" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="banking.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/banking" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="green" maxmem="4063" memory="400" name="banking" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="7" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="untrusted.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/untrusted" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="red" maxmem="4063" memory="400" name="untrusted" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="8" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="test-standalonevm.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/test-standalonevm" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="None" kernelopts="" label="red" maxmem="4063" memory="400" name="test-standalonevm" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="9" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="none" uses_default_kernel="False" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="test-testvm.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/test-testvm" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="red" mac="00:16:3E:5E:6C:55" maxmem="4063" memory="400" name="test-testvm" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="10" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesTemplateVm conf_file="test-template-clone.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/vm-templates/test-template-clone" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="gray" maxmem="4063" memory="400" name="test-template-clone" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="11" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="none" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="test-custom-template-appvm.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/test-custom-template-appvm" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="yellow" maxmem="4063" memory="400" name="test-custom-template-appvm" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="12" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="11" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesProxyVm conf_file="test-testproxy.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/servicevms/test-testproxy" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="yellow" maxmem="4063" memory="200" name="test-testproxy" netid="3" netvm_qid="2" pcidevs="[]" private_img="private.img" qid="13" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" vcpus="2" volatile_img="volatile.img" /></QubesVmCollection>
  117. '''
  118. BACKUP_HEADER_R2 = '''version=3
  119. hmac-algorithm=SHA512
  120. crypto-algorithm=aes-256-cbc
  121. encrypted={encrypted}
  122. compressed={compressed}
  123. compression-filter=gzip
  124. '''
  125. BACKUP_HEADER_R4 = '''version=4
  126. hmac-algorithm=scrypt
  127. encrypted=True
  128. compressed={compressed}
  129. compression-filter={compression_filter}
  130. backup-id=20161020T123455-1234
  131. '''
  132. parsed_qubes_xml_r2 = {
  133. 'domains': {
  134. 'dom0': {
  135. 'klass': 'AdminVM',
  136. 'label': 'black',
  137. 'properties': {},
  138. 'devices': {},
  139. 'tags': set(),
  140. 'features': {},
  141. 'template': None,
  142. 'backup_path': 'dom0-home/user',
  143. 'included_in_backup': True,
  144. },
  145. 'fedora-20-x64': {
  146. 'klass': 'TemplateVM',
  147. 'label': 'black',
  148. 'properties': {
  149. 'maxmem': '1535',
  150. },
  151. 'devices': {},
  152. 'tags': set(),
  153. 'features': {
  154. 'service.meminfo-writer': True,
  155. 'qrexec': True,
  156. 'gui': True,
  157. },
  158. 'template': None,
  159. 'backup_path': None,
  160. 'included_in_backup': False,
  161. },
  162. 'netvm': {
  163. 'klass': 'AppVM',
  164. 'label': 'red',
  165. 'properties': {
  166. 'maxmem': '1535',
  167. 'memory': '200',
  168. 'netvm': None,
  169. 'default_dispvm': 'disp-no-netvm',
  170. 'provides_network': True},
  171. 'devices': {
  172. 'pci': {
  173. ('dom0', '02_00.0'): {},
  174. ('dom0', '03_00.0'): {},
  175. }
  176. },
  177. 'tags': set(),
  178. 'features': {
  179. 'service.ntpd': False,
  180. 'service.meminfo-writer': False
  181. },
  182. 'template': 'fedora-20-x64',
  183. 'backup_path': None,
  184. 'included_in_backup': False,
  185. },
  186. 'firewallvm': {
  187. 'klass': 'AppVM',
  188. 'label': 'green',
  189. 'properties': {
  190. 'maxmem': '1535',
  191. 'memory': '200',
  192. 'provides_network': True
  193. },
  194. 'devices': {},
  195. 'tags': set(),
  196. 'features': {'service.meminfo-writer': True},
  197. 'template': 'fedora-20-x64',
  198. 'backup_path': None,
  199. 'included_in_backup': False,
  200. },
  201. 'fedora-20-x64-dvm': {
  202. 'klass': 'AppVM',
  203. 'label': 'gray',
  204. 'properties': {
  205. 'maxmem': '1535',
  206. 'vcpus': '1'
  207. },
  208. 'devices': {},
  209. 'tags': set(),
  210. 'features': {
  211. 'internal': True, 'service.meminfo-writer': True},
  212. 'template': 'fedora-20-x64',
  213. 'backup_path': None,
  214. 'included_in_backup': False,
  215. },
  216. 'banking': {
  217. 'klass': 'AppVM',
  218. 'label': 'green',
  219. 'properties': {'maxmem': '1535'},
  220. 'devices': {},
  221. 'tags': set(),
  222. 'features': {'service.meminfo-writer': True},
  223. 'template': 'fedora-20-x64',
  224. 'backup_path': None,
  225. 'included_in_backup': False,
  226. },
  227. 'personal': {
  228. 'klass': 'AppVM',
  229. 'label': 'yellow',
  230. 'properties': {'maxmem': '1535'},
  231. 'devices': {},
  232. 'tags': set(),
  233. 'features': {'service.meminfo-writer': True},
  234. 'template': 'fedora-20-x64',
  235. 'backup_path': None,
  236. 'included_in_backup': False,
  237. },
  238. 'untrusted': {
  239. 'klass': 'AppVM',
  240. 'label': 'red',
  241. 'properties': {
  242. 'maxmem': '1535',
  243. 'netvm': 'test-testproxy',
  244. 'default_dispvm': 'disp-test-testproxy',
  245. },
  246. 'devices': {},
  247. 'tags': set(),
  248. 'features': {'service.meminfo-writer': True},
  249. 'template': 'fedora-20-x64',
  250. 'backup_path': None,
  251. 'included_in_backup': False,
  252. },
  253. 'testproxy2': {
  254. 'klass': 'AppVM',
  255. 'label': 'red',
  256. 'properties': {
  257. 'maxmem': '1535',
  258. 'memory': '200',
  259. 'provides_network': True},
  260. 'devices': {},
  261. 'tags': set(),
  262. 'features': {'service.meminfo-writer': True},
  263. 'template': 'test-template-clone',
  264. 'backup_path': None,
  265. 'included_in_backup': False,
  266. },
  267. 'test-testproxy': {
  268. 'klass': 'AppVM',
  269. 'label': 'red',
  270. 'properties': {
  271. 'maxmem': '1535',
  272. 'memory': '200',
  273. 'provides_network': True},
  274. 'devices': {},
  275. 'tags': set(),
  276. 'features': {'service.meminfo-writer': True},
  277. 'template': 'fedora-20-x64',
  278. 'backup_path': 'servicevms/test-testproxy',
  279. 'included_in_backup': True,
  280. },
  281. 'test-testhvm': {
  282. 'klass': 'StandaloneVM',
  283. 'label': 'purple',
  284. 'properties': {'kernel': '', 'virt_mode': 'hvm', 'memory': '512'},
  285. 'devices': {},
  286. 'tags': set(),
  287. 'features': {
  288. 'service.meminfo-writer': False,
  289. 'linux-stubdom': False},
  290. 'template': None,
  291. 'backup_path': 'appvms/test-testhvm',
  292. 'included_in_backup': True,
  293. 'root_size': 2097664,
  294. },
  295. 'test-work': {
  296. 'klass': 'AppVM',
  297. 'label': 'green',
  298. 'properties': {'maxmem': '1535'},
  299. 'devices': {},
  300. 'tags': set(),
  301. 'features': {'service.meminfo-writer': True},
  302. 'template': 'fedora-20-x64',
  303. 'backup_path': 'appvms/test-work',
  304. 'included_in_backup': True,
  305. },
  306. 'test-template-clone': {
  307. 'klass': 'TemplateVM',
  308. 'label': 'green',
  309. 'properties': {'maxmem': '1535'},
  310. 'devices': {},
  311. 'tags': set(),
  312. 'features': {
  313. 'service.meminfo-writer': True,
  314. 'qrexec': True,
  315. 'gui': True,
  316. },
  317. 'template': None,
  318. 'backup_path': 'vm-templates/test-template-clone',
  319. 'included_in_backup': True,
  320. 'root_size': 209715712,
  321. },
  322. 'test-custom-template-appvm': {
  323. 'klass': 'AppVM',
  324. 'label': 'yellow',
  325. 'properties': {'maxmem': '1535'},
  326. 'devices': {},
  327. 'tags': set(),
  328. 'features': {'service.meminfo-writer': True},
  329. 'template': 'test-template-clone',
  330. 'backup_path': 'appvms/test-custom-template-appvm',
  331. 'included_in_backup': True,
  332. },
  333. 'test-standalonevm': {
  334. 'klass': 'StandaloneVM',
  335. 'label': 'blue',
  336. 'properties': {'maxmem': '1535'},
  337. 'devices': {},
  338. 'tags': set(),
  339. 'features': {
  340. 'service.meminfo-writer': True,
  341. 'qrexec': True,
  342. 'gui': True,
  343. },
  344. 'template': None,
  345. 'backup_path': 'appvms/test-standalonevm',
  346. 'included_in_backup': True,
  347. 'root_size': 2097664,
  348. },
  349. 'test-net': {
  350. 'klass': 'AppVM',
  351. 'label': 'red',
  352. 'properties': {
  353. 'maxmem': '1535',
  354. 'memory': '200',
  355. 'netvm': None,
  356. 'default_dispvm': 'disp-no-netvm',
  357. 'provides_network': True},
  358. 'devices': {
  359. 'pci': {
  360. ('dom0', '02_00.0'): {},
  361. ('dom0', '03_00.0'): {},
  362. }
  363. },
  364. 'tags': set(),
  365. 'features': {
  366. 'service.ntpd': False,
  367. 'service.meminfo-writer': False
  368. },
  369. 'template': 'fedora-20-x64',
  370. 'backup_path': 'servicevms/test-net',
  371. 'included_in_backup': True,
  372. },
  373. 'disp-no-netvm': {
  374. 'klass': 'AppVM',
  375. 'label': 'red',
  376. 'properties': {
  377. 'netvm': None,
  378. 'template_for_dispvms': True,
  379. },
  380. 'devices': {},
  381. 'features': {},
  382. 'tags': set(),
  383. 'template': None, # default
  384. 'backup_path': None,
  385. 'included_in_backup': True,
  386. },
  387. 'disp-test-testproxy': {
  388. 'klass': 'AppVM',
  389. 'label': 'red',
  390. 'properties': {
  391. 'netvm': 'test-testproxy',
  392. 'template_for_dispvms': True,
  393. },
  394. 'devices': {},
  395. 'features': {},
  396. 'tags': set(),
  397. 'template': None, # default
  398. 'backup_path': None,
  399. 'included_in_backup': True,
  400. },
  401. },
  402. 'globals': {
  403. 'default_template': 'fedora-20-x64',
  404. 'default_kernel': '3.7.6-2',
  405. 'default_netvm': 'firewallvm',
  406. 'clockvm': 'netvm',
  407. 'updatevm': 'firewallvm'
  408. },
  409. }
  410. parsed_qubes_xml_v4 = {
  411. 'domains': {
  412. 'dom0': {
  413. 'klass': 'AdminVM',
  414. 'label': 'black',
  415. 'properties': {},
  416. 'devices': {},
  417. 'tags': set(),
  418. 'features': {},
  419. 'template': None,
  420. 'backup_path': 'dom0-home/user',
  421. 'included_in_backup': True,
  422. },
  423. 'fedora-25': {
  424. 'klass': 'TemplateVM',
  425. 'label': 'black',
  426. 'properties': {},
  427. 'devices': {},
  428. 'tags': {'created-by-test-work'},
  429. 'features': {
  430. 'gui': '1',
  431. 'qrexec': 'True',
  432. 'updates-available': False
  433. },
  434. 'template': None,
  435. 'backup_path': None,
  436. 'included_in_backup': False,
  437. },
  438. 'fedora-25-lvm': {
  439. 'klass': 'TemplateVM',
  440. 'label': 'black',
  441. 'properties': {
  442. 'maxmem': '4000',
  443. },
  444. 'devices': {},
  445. 'tags': set(),
  446. 'features': {},
  447. 'template': None,
  448. 'backup_path': None,
  449. 'included_in_backup': False,
  450. },
  451. 'debian-8': {
  452. 'klass': 'TemplateVM',
  453. 'label': 'black',
  454. 'properties': {},
  455. 'devices': {},
  456. 'tags': {'created-by-dom0'},
  457. 'features': {
  458. 'gui': '1',
  459. 'qrexec': 'True',
  460. 'updates-available': False},
  461. 'template': None,
  462. 'backup_path': None,
  463. 'included_in_backup': False,
  464. },
  465. 'sys-net': {
  466. 'klass': 'AppVM',
  467. 'label': 'red',
  468. 'properties': {
  469. 'virt_mode': 'hvm',
  470. 'kernelopts': 'nopat i8042.nokbd i8042.noaux',
  471. 'maxmem': '300',
  472. 'memory': '300',
  473. 'netvm': None,
  474. 'default_user': 'user',
  475. 'provides_network': 'True'},
  476. 'devices': {
  477. 'pci': {
  478. ('dom0', '02_00.0'): {},
  479. }
  480. },
  481. 'tags': set(),
  482. 'features': {
  483. 'service.clocksync': '1',
  484. 'service.meminfo-writer': False
  485. },
  486. 'template': 'fedora-25',
  487. 'backup_path': None,
  488. 'included_in_backup': False,
  489. },
  490. 'sys-firewall': {
  491. 'klass': 'AppVM',
  492. 'label': 'green',
  493. 'properties': {
  494. 'autostart': 'True',
  495. 'memory': '500',
  496. 'provides_network': 'True'
  497. },
  498. 'devices': {},
  499. 'tags': set(),
  500. 'features': {},
  501. 'template': 'fedora-25',
  502. 'backup_path': None,
  503. 'included_in_backup': False,
  504. },
  505. 'test-d8test': {
  506. 'klass': 'AppVM',
  507. 'label': 'gray',
  508. 'properties': {'debug': 'True', 'kernel': None},
  509. 'devices': {},
  510. 'tags': {'created-by-dom0'},
  511. 'features': {},
  512. 'template': 'debian-8',
  513. 'backup_path': 'appvms/test-d8test',
  514. 'included_in_backup': True,
  515. },
  516. 'fedora-25-dvm': {
  517. 'klass': 'AppVM',
  518. 'label': 'red',
  519. 'properties': {
  520. 'template_for_dispvms': 'True',
  521. 'vcpus': '1',
  522. },
  523. 'devices': {},
  524. 'tags': set(),
  525. 'features': {
  526. 'internal': '1', 'service.meminfo-writer': '1'},
  527. 'template': 'fedora-25',
  528. 'backup_path': None,
  529. 'included_in_backup': False,
  530. },
  531. 'fedora-25-clone-dvm': {
  532. 'klass': 'AppVM',
  533. 'label': 'red',
  534. 'properties': {
  535. 'vcpus': '1',
  536. 'template_for_dispvms': 'True',
  537. },
  538. 'devices': {},
  539. 'tags': set(),
  540. 'features': {
  541. 'internal': '1', 'service.meminfo-writer': '1'},
  542. 'template': 'test-fedora-25-clone',
  543. 'backup_path': None,
  544. 'included_in_backup': False,
  545. },
  546. 'vault': {
  547. 'klass': 'AppVM',
  548. 'label': 'black',
  549. 'properties': {'virt_mode': 'pv', 'maxmem': '1536', 'netvm': None},
  550. 'devices': {},
  551. 'tags': set(),
  552. 'features': {},
  553. 'template': 'fedora-25',
  554. 'backup_path': None,
  555. 'included_in_backup': False,
  556. },
  557. 'personal': {
  558. 'klass': 'AppVM',
  559. 'label': 'yellow',
  560. 'properties': {'netvm': 'sys-firewall'},
  561. 'devices': {},
  562. 'tags': set(),
  563. 'features': {
  564. 'feat1': '1',
  565. 'feat2': False,
  566. 'feat32': '1',
  567. 'featdis': False,
  568. 'xxx': '1'
  569. },
  570. 'template': 'fedora-25',
  571. 'backup_path': None,
  572. 'included_in_backup': False,
  573. },
  574. 'untrusted': {
  575. 'klass': 'AppVM',
  576. 'label': 'red',
  577. 'properties': {
  578. 'netvm': None,
  579. 'backup_timestamp': '1474318497',
  580. 'default_dispvm': 'fedora-25-clone-dvm',
  581. },
  582. 'devices': {},
  583. 'tags': set(),
  584. 'features': {'service.meminfo-writer': '1'},
  585. 'template': 'fedora-25',
  586. 'backup_path': None,
  587. 'included_in_backup': False,
  588. },
  589. 'sys-usb': {
  590. 'klass': 'AppVM',
  591. 'label': 'red',
  592. 'properties': {
  593. 'autostart': 'True',
  594. 'maxmem': '400',
  595. 'provides_network': 'True',
  596. },
  597. 'devices': {},
  598. 'tags': set(),
  599. 'features': {
  600. 'service.meminfo-writer': False,
  601. 'service.network-manager': False,
  602. },
  603. 'template': 'fedora-25',
  604. 'backup_path': None,
  605. 'included_in_backup': False,
  606. },
  607. 'test-proxy': {
  608. 'klass': 'AppVM',
  609. 'label': 'red',
  610. 'properties': {'netvm': 'sys-net', 'provides_network': 'True'},
  611. 'devices': {},
  612. 'tags': {'created-by-dom0'},
  613. 'features': {},
  614. 'template': 'debian-8',
  615. 'backup_path': 'appvms/test-proxy',
  616. 'included_in_backup': True,
  617. },
  618. 'test-hvm': {
  619. 'klass': 'StandaloneVM',
  620. 'label': 'purple',
  621. 'properties': {
  622. 'kernel': None,
  623. 'virt_mode': 'hvm',
  624. 'maxmem': '4000'},
  625. 'devices': {},
  626. 'tags': set(),
  627. 'features': {'service.meminfo-writer': False},
  628. 'template': None,
  629. 'backup_path': 'appvms/test-hvm',
  630. 'included_in_backup': True,
  631. 'root_size': 2097664,
  632. },
  633. 'test-work': {
  634. 'klass': 'AppVM',
  635. 'label': 'green',
  636. 'properties': {
  637. 'ip': '192.168.0.1',
  638. 'maxmem': '4000',
  639. 'memory': '400'},
  640. 'devices': {},
  641. 'tags': {'tag1', 'tag2'},
  642. 'features': {'service.meminfo-writer': '1'},
  643. 'template': 'fedora-25',
  644. 'backup_path': 'appvms/test-work',
  645. 'included_in_backup': True,
  646. },
  647. 'test-fedora-25-clone': {
  648. 'klass': 'TemplateVM',
  649. 'label': 'black',
  650. 'properties': {'maxmem': '4000'},
  651. 'devices': {},
  652. 'tags': set(),
  653. 'features': {'service.meminfo-writer': '1'},
  654. 'template': None,
  655. 'backup_path': 'vm-templates/test-fedora-25-clone',
  656. 'included_in_backup': True,
  657. },
  658. 'test-custom-template-appvm': {
  659. 'klass': 'AppVM',
  660. 'label': 'yellow',
  661. 'properties': {'debug': 'True', 'kernel': None},
  662. 'devices': {},
  663. 'tags': {'created-by-dom0'},
  664. 'features': {},
  665. 'template': 'test-fedora-25-clone',
  666. 'backup_path': 'appvms/test-custom-template-appvm',
  667. 'included_in_backup': True,
  668. },
  669. 'test-standalonevm': {
  670. 'klass': 'StandaloneVM',
  671. 'label': 'blue',
  672. 'properties': {'maxmem': '4000'},
  673. 'devices': {},
  674. 'tags': set(),
  675. 'features': {},
  676. 'template': None,
  677. 'backup_path': 'appvms/test-standalonevm',
  678. 'included_in_backup': True,
  679. 'root_size': 2097664,
  680. },
  681. 'test-net': {
  682. 'klass': 'AppVM',
  683. 'label': 'red',
  684. 'properties': {
  685. 'maxmem': '300',
  686. 'memory': '300',
  687. 'netvm': None,
  688. 'provides_network': 'True'
  689. },
  690. 'devices': {
  691. 'pci': {
  692. ('dom0', '03_00.0'): {},
  693. }
  694. },
  695. 'tags': set(),
  696. 'features': {
  697. 'service.ntpd': False,
  698. 'service.meminfo-writer': False
  699. },
  700. 'template': 'fedora-25',
  701. 'backup_path': 'appvms/test-net',
  702. 'included_in_backup': True,
  703. },
  704. },
  705. 'globals': {
  706. 'default_template': 'fedora-25',
  707. 'default_kernel': '4.9.31-17',
  708. 'default_netvm': 'sys-firewall',
  709. 'default_dispvm': 'fedora-25-dvm',
  710. #'default_fw_netvm': 'sys-net',
  711. 'clockvm': 'sys-net',
  712. 'updatevm': 'sys-firewall'
  713. },
  714. }
  715. class TC_00_QubesXML(qubesadmin.tests.QubesTestCase):
  716. def assertCorrectlyConverted(self, backup_app, expected_data):
  717. self.assertCountEqual(backup_app.domains.keys(),
  718. expected_data['domains'].keys())
  719. for vm in expected_data['domains']:
  720. self.assertEqual(backup_app.domains[vm].name, vm)
  721. self.assertEqual(backup_app.domains[vm].properties,
  722. expected_data['domains'][vm]['properties'], vm)
  723. for devtype in expected_data['domains'][vm]['devices']:
  724. self.assertEqual(backup_app.domains[vm].devices[devtype],
  725. expected_data['domains'][vm]['devices'][devtype], vm)
  726. self.assertEqual(backup_app.domains[vm].tags,
  727. expected_data['domains'][vm]['tags'], vm)
  728. self.assertEqual(backup_app.domains[vm].features,
  729. expected_data['domains'][vm]['features'], vm)
  730. self.assertEqual(backup_app.domains[vm].template,
  731. expected_data['domains'][vm]['template'], vm)
  732. self.assertEqual(backup_app.domains[vm].klass,
  733. expected_data['domains'][vm]['klass'], vm)
  734. self.assertEqual(backup_app.domains[vm].label,
  735. expected_data['domains'][vm]['label'], vm)
  736. self.assertEqual(backup_app.domains[vm].backup_path,
  737. expected_data['domains'][vm]['backup_path'], vm)
  738. self.assertEqual(backup_app.domains[vm].included_in_backup,
  739. expected_data['domains'][vm]['included_in_backup'], vm)
  740. self.assertEqual(backup_app.globals, expected_data['globals'])
  741. def test_000_qubes_xml_r2(self):
  742. xml_data = pkg_resources.resource_string(__name__, 'v3-qubes.xml')
  743. with tempfile.NamedTemporaryFile() as qubes_xml:
  744. qubes_xml.file.write(xml_data)
  745. backup_app = qubesadmin.backup.core2.Core2Qubes(qubes_xml.name)
  746. self.assertCorrectlyConverted(backup_app, parsed_qubes_xml_r2)
  747. def test_010_qubes_xml_r4(self):
  748. self.maxDiff = None
  749. xml_data = pkg_resources.resource_string(__name__, 'v4-qubes.xml')
  750. with tempfile.NamedTemporaryFile() as qubes_xml:
  751. qubes_xml.file.write(xml_data)
  752. backup_app = qubesadmin.backup.core3.Core3Qubes(qubes_xml.name)
  753. self.assertCorrectlyConverted(backup_app, parsed_qubes_xml_v4)
  754. # backup code use multiprocessing, synchronize with main process
  755. class AppProxy(object):
  756. def __init__(self, app, sync_queue):
  757. self._app = app
  758. self._sync_queue = sync_queue
  759. def qubesd_call(self, dest, method, arg=None, payload=None,
  760. payload_stream=None):
  761. if payload_stream:
  762. signature_bin = payload_stream.read(512)
  763. payload = signature_bin.split(b'\0', 1)[0]
  764. subprocess.call(['cat'], stdin=payload_stream,
  765. stdout=subprocess.DEVNULL)
  766. payload_stream.close()
  767. self._sync_queue.put((dest, method, arg, payload))
  768. return self._app.qubesd_call(dest, method, arg, payload)
  769. class MockVolume(qubesadmin.storage.Volume):
  770. def __init__(self, import_data_queue, *args, **kwargs):
  771. super(MockVolume, self).__init__(*args, **kwargs)
  772. self.app = AppProxy(self.app, import_data_queue)
  773. class MockFirewall(qubesadmin.firewall.Firewall):
  774. def __init__(self, import_data_queue, *args, **kwargs):
  775. super(MockFirewall, self).__init__(*args, **kwargs)
  776. self.vm.app = AppProxy(self.vm.app, import_data_queue)
  777. @unittest.skipUnless(os.environ.get('ENABLE_SLOW_TESTS', False),
  778. 'Set ENABLE_SLOW_TESTS=1 environment variable')
  779. class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
  780. storage_pool = None
  781. def tearDown(self):
  782. super(TC_10_BackupCompatibility, self).tearDown()
  783. def create_whitelisted_appmenus(self, filename):
  784. with open(filename, "w") as f:
  785. f.write("gnome-terminal.desktop\n")
  786. f.write("nautilus.desktop\n")
  787. f.write("firefox.desktop\n")
  788. f.write("mozilla-thunderbird.desktop\n")
  789. f.write("libreoffice-startcenter.desktop\n")
  790. def create_appmenus(self, dir, template, list):
  791. for name in list:
  792. with open(os.path.join(dir, name + ".desktop"), "w") as f:
  793. f.write(template.format(name=name, comment=name, command=name))
  794. def create_private_img(self, filename):
  795. signature = '/'.join(os.path.splitext(filename)[0].split('/')[-2:])
  796. self.create_sparse(filename, 2*2**20, signature=signature.encode())
  797. #subprocess.check_call(["/usr/sbin/mkfs.ext4", "-q", "-F", filename])
  798. def create_volatile_img(self, filename):
  799. self.create_sparse(filename, 11.5*2**20)
  800. # here used to be sfdisk call with "0,1024,S\n,10240,L\n" input,
  801. # but since sfdisk folks like to change command arguments in
  802. # incompatible way, have an partition table verbatim here
  803. ptable = (
  804. '\x00\x00\x00\x00\x00\x00\x00\x00\xab\x39\xd5\xd4\x00\x00\x20\x00'
  805. '\x00\x21\xaa\x82\x82\x28\x08\x00\x00\x00\x00\x00\x00\x20\xaa\x00'
  806. '\x82\x29\x15\x83\x9c\x79\x08\x00\x00\x20\x00\x00\x01\x40\x00\x00'
  807. '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
  808. '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xaa\x55'
  809. )
  810. with open(filename, 'r+') as f:
  811. f.seek(0x1b0)
  812. f.write(ptable)
  813. # TODO: mkswap
  814. def fullpath(self, name):
  815. return os.path.join(self.backupdir, name)
  816. def create_v1_files(self, r2b2=False):
  817. appmenus_list = [
  818. "firefox", "gnome-terminal", "evince", "evolution",
  819. "mozilla-thunderbird", "libreoffice-startcenter", "nautilus",
  820. "gedit", "gpk-update-viewer", "gpk-application"
  821. ]
  822. os.mkdir(self.fullpath("appvms"))
  823. os.mkdir(self.fullpath("servicevms"))
  824. os.mkdir(self.fullpath("vm-templates"))
  825. # normal AppVM, with firewall
  826. os.mkdir(self.fullpath("appvms/test-work"))
  827. self.create_whitelisted_appmenus(self.fullpath(
  828. "appvms/test-work/whitelisted-appmenus.list"))
  829. os.symlink("/usr/share/qubes/icons/green.png",
  830. self.fullpath("appvms/test-work/icon.png"))
  831. self.create_private_img(self.fullpath("appvms/test-work/private.img"))
  832. with open(self.fullpath("appvms/test-work/firewall.xml"), "wb") as \
  833. f_firewall:
  834. f_firewall.write(
  835. pkg_resources.resource_string(__name__, 'v3-firewall.xml'))
  836. # StandaloneVM
  837. os.mkdir(self.fullpath("appvms/test-standalonevm"))
  838. self.create_whitelisted_appmenus(self.fullpath(
  839. "appvms/test-standalonevm/whitelisted-appmenus.list"))
  840. os.symlink("/usr/share/qubes/icons/blue.png",
  841. self.fullpath("appvms/test-standalonevm/icon.png"))
  842. self.create_private_img(self.fullpath(
  843. "appvms/test-standalonevm/private.img"))
  844. self.create_sparse(
  845. self.fullpath("appvms/test-standalonevm/root.img"), 10*2**30)
  846. self.fill_image(self.fullpath("appvms/test-standalonevm/root.img"),
  847. 1024*1024, True,
  848. signature=b'test-standalonevm/root')
  849. os.mkdir(self.fullpath("appvms/test-standalonevm/apps.templates"))
  850. self.create_appmenus(self.fullpath("appvms/test-standalonevm/apps"
  851. ".templates"),
  852. APPTEMPLATE_R2B2,
  853. appmenus_list)
  854. os.mkdir(self.fullpath("appvms/test-standalonevm/kernels"))
  855. for k_file in ["initramfs", "vmlinuz", "modules.img"]:
  856. self.fill_image(self.fullpath("appvms/test-standalonevm/kernels/"
  857. + k_file), 1024*1024)
  858. # VM based on custom template
  859. subprocess.check_call(
  860. ["/bin/cp", "-a", self.fullpath("appvms/test-work"),
  861. self.fullpath("appvms/test-custom-template-appvm")])
  862. # override for correct signature
  863. self.create_private_img(
  864. self.fullpath("appvms/test-custom-template-appvm/private.img"))
  865. # HVM
  866. if r2b2:
  867. subprocess.check_call(
  868. ["/bin/cp", "-a", self.fullpath("appvms/test-standalonevm"),
  869. self.fullpath("appvms/test-testhvm")])
  870. # override for correct signature
  871. self.create_private_img(
  872. self.fullpath("appvms/test-testhvm/private.img"))
  873. self.fill_image(self.fullpath("appvms/test-testhvm/root.img"),
  874. 1024*1024, True,
  875. signature=b'test-testhvm/root')
  876. # ProxyVM
  877. os.mkdir(self.fullpath("servicevms/test-testproxy"))
  878. self.create_whitelisted_appmenus(self.fullpath(
  879. "servicevms/test-testproxy/whitelisted-appmenus.list"))
  880. self.create_private_img(
  881. self.fullpath("servicevms/test-testproxy/private.img"))
  882. # NetVM
  883. os.mkdir(self.fullpath("servicevms/test-net"))
  884. self.create_whitelisted_appmenus(self.fullpath(
  885. "servicevms/test-net/whitelisted-appmenus.list"))
  886. self.create_private_img(
  887. self.fullpath("servicevms/test-net/private.img"))
  888. # Custom template
  889. os.mkdir(self.fullpath("vm-templates/test-template-clone"))
  890. self.create_private_img(
  891. self.fullpath("vm-templates/test-template-clone/private.img"))
  892. self.create_sparse(self.fullpath(
  893. "vm-templates/test-template-clone/root-cow.img"), 10*2**20)
  894. self.create_sparse(self.fullpath(
  895. "vm-templates/test-template-clone/root.img"), 10*2**20)
  896. self.fill_image(self.fullpath(
  897. "vm-templates/test-template-clone/root.img"), 100*2**20, True,
  898. signature=b'test-template-clone/root')
  899. self.create_volatile_img(self.fullpath(
  900. "vm-templates/test-template-clone/volatile.img"))
  901. subprocess.check_call([
  902. "/bin/tar", "cS",
  903. "-f", self.fullpath(
  904. "vm-templates/test-template-clone/clean-volatile.img.tar"),
  905. "-C", self.fullpath("vm-templates/test-template-clone"),
  906. "volatile.img"])
  907. self.create_whitelisted_appmenus(self.fullpath(
  908. "vm-templates/test-template-clone/whitelisted-appmenus.list"))
  909. self.create_whitelisted_appmenus(self.fullpath(
  910. "vm-templates/test-template-clone/vm-whitelisted-appmenus.list"))
  911. if r2b2:
  912. self.create_whitelisted_appmenus(self.fullpath(
  913. "vm-templates/test-template-clone/netvm-whitelisted-appmenus"
  914. ".list"))
  915. os.symlink("/usr/share/qubes/icons/green.png",
  916. self.fullpath("vm-templates/test-template-clone/icon.png"))
  917. os.mkdir(
  918. self.fullpath("vm-templates/test-template-clone/apps.templates"))
  919. self.create_appmenus(
  920. self.fullpath("vm-templates/test-template-clone/apps.templates"),
  921. APPTEMPLATE_R2B2,
  922. appmenus_list)
  923. os.mkdir(self.fullpath("vm-templates/test-template-clone/apps"))
  924. self.create_appmenus(
  925. self.fullpath("vm-templates/test-template-clone/apps"),
  926. APPTEMPLATE_R2B2.replace("%VMNAME%", "test-template-clone")
  927. .replace("%VMDIR%", self.fullpath(
  928. "vm-templates/test-template-clone")),
  929. appmenus_list)
  930. self.create_dom0_files()
  931. dom0_dirs = ('Downloads', 'Pictures', 'Documents', '.config', '.local')
  932. dom0_files = ('.bash_history', 'some-file.txt',
  933. 'Pictures/another-file.png')
  934. def create_dom0_files(self):
  935. # dom0 files
  936. os.mkdir(self.fullpath('dom0-home'))
  937. os.mkdir(self.fullpath('dom0-home/user'))
  938. for d in self.dom0_dirs:
  939. os.mkdir(self.fullpath('dom0-home/user/' + d))
  940. for f in self.dom0_files:
  941. with open(self.fullpath('dom0-home/user/' + f), 'w') as ff:
  942. ff.write('some content')
  943. def assertDirectoryExists(self, path):
  944. if not os.path.exists(path):
  945. self.fail(path + ' missing')
  946. if not os.path.isdir(path):
  947. self.fail(path + ' is not a directory')
  948. def assertFileExists(self, path):
  949. if not os.path.exists(path):
  950. self.fail(path + ' missing')
  951. if not os.path.isfile(path):
  952. self.fail(path + ' is not a file')
  953. def assertDom0Restored(self, timestamp):
  954. expected_dir = os.path.expanduser(
  955. '~/home-restore-' + timestamp + '/dom0-home/user')
  956. self.assertTrue(os.path.exists(expected_dir))
  957. for d in self.dom0_dirs:
  958. self.assertDirectoryExists(os.path.join(expected_dir, d))
  959. for f in self.dom0_files:
  960. self.assertFileExists(os.path.join(expected_dir, f))
  961. # cleanup
  962. shutil.rmtree(expected_dir)
  963. def create_v4_files(self):
  964. appmenus_list = [
  965. "firefox", "gnome-terminal", "evince", "evolution",
  966. "mozilla-thunderbird", "libreoffice-startcenter", "nautilus",
  967. "gedit", "gpk-update-viewer", "gpk-application"
  968. ]
  969. os.mkdir(self.fullpath("appvms"))
  970. os.mkdir(self.fullpath("vm-templates"))
  971. # normal AppVMs
  972. for vm in ('test-work', 'test-d8test', 'test-proxy',
  973. 'test-custom-template-appvm', 'test-net'):
  974. os.mkdir(self.fullpath('appvms/{}'.format(vm)))
  975. self.create_whitelisted_appmenus(self.fullpath(
  976. 'appvms/{}/whitelisted-appmenus.list'.format(vm)))
  977. self.create_private_img(self.fullpath('appvms/{}/private.img'.format(
  978. vm)))
  979. # setup firewall only on one VM
  980. with open(self.fullpath("appvms/test-work/firewall.xml"), "wb") as \
  981. f_firewall:
  982. f_firewall.write(
  983. pkg_resources.resource_string(__name__, 'v4-firewall.xml'))
  984. # StandaloneVMs
  985. for vm in ('test-standalonevm', 'test-hvm'):
  986. os.mkdir(self.fullpath('appvms/{}'.format(vm)))
  987. self.create_whitelisted_appmenus(self.fullpath(
  988. 'appvms/{}/whitelisted-appmenus.list'.format(vm)))
  989. self.create_private_img(self.fullpath(
  990. 'appvms/{}/private.img'.format(vm)))
  991. self.create_sparse(
  992. self.fullpath('appvms/{}/root.img'.format(vm)), 10*2**30)
  993. self.fill_image(self.fullpath('appvms/{}/root.img'.format(vm)),
  994. 1024*1024, True,
  995. signature='{}/root'.format(vm).encode())
  996. # only for Linux one
  997. os.mkdir(self.fullpath('appvms/test-standalonevm/apps.templates'))
  998. self.create_appmenus(
  999. self.fullpath('appvms/test-standalonevm/apps.templates'),
  1000. APPTEMPLATE_R2B2,
  1001. appmenus_list)
  1002. # Custom template
  1003. os.mkdir(self.fullpath("vm-templates/test-fedora-25-clone"))
  1004. self.create_private_img(
  1005. self.fullpath("vm-templates/test-fedora-25-clone/private.img"))
  1006. self.create_sparse(self.fullpath(
  1007. "vm-templates/test-fedora-25-clone/root.img"), 10*2**20)
  1008. self.fill_image(self.fullpath(
  1009. "vm-templates/test-fedora-25-clone/root.img"), 1*2**20, True,
  1010. signature=b'test-fedora-25-clone/root')
  1011. self.create_volatile_img(self.fullpath(
  1012. "vm-templates/test-fedora-25-clone/volatile.img"))
  1013. self.create_whitelisted_appmenus(self.fullpath(
  1014. "vm-templates/test-fedora-25-clone/whitelisted-appmenus.list"))
  1015. self.create_whitelisted_appmenus(self.fullpath(
  1016. "vm-templates/test-fedora-25-clone/vm-whitelisted-appmenus.list"))
  1017. os.mkdir(
  1018. self.fullpath("vm-templates/test-fedora-25-clone/apps.templates"))
  1019. self.create_appmenus(
  1020. self.fullpath("vm-templates/test-fedora-25-clone/apps.templates"),
  1021. APPTEMPLATE_R2B2,
  1022. appmenus_list)
  1023. os.mkdir(self.fullpath("vm-templates/test-fedora-25-clone/apps"))
  1024. self.create_appmenus(
  1025. self.fullpath("vm-templates/test-fedora-25-clone/apps"),
  1026. APPTEMPLATE_R2B2.replace("%VMNAME%", "test-fedora-25-clone")
  1027. .replace("%VMDIR%", self.fullpath(
  1028. "vm-templates/test-fedora-25-clone")),
  1029. appmenus_list)
  1030. self.create_dom0_files()
  1031. def scrypt_encrypt(self, f_name, output_name=None, password='qubes',
  1032. basedir=None):
  1033. if basedir is None:
  1034. basedir = self.backupdir
  1035. if output_name is None:
  1036. output_name = f_name + '.enc'
  1037. if f_name == 'backup-header':
  1038. scrypt_pass = 'backup-header!' + password
  1039. else:
  1040. scrypt_pass = '20161020T123455-1234!{}!{}'.format(f_name, password)
  1041. p = subprocess.Popen(['scrypt', 'enc', '-P', '-t', '0.1',
  1042. os.path.join(basedir, f_name), os.path.join(basedir, output_name)],
  1043. stdin=subprocess.PIPE)
  1044. p.communicate(scrypt_pass.encode())
  1045. assert p.wait() == 0
  1046. return output_name
  1047. def calculate_hmac(self, f_name, algorithm="sha512", password="qubes"):
  1048. with open(self.fullpath(f_name), "r") as f_data:
  1049. with open(self.fullpath(f_name+".hmac"), "w") as f_hmac:
  1050. subprocess.check_call(
  1051. ["openssl", "dgst", "-"+algorithm, "-hmac", password],
  1052. stdin=f_data, stdout=f_hmac)
  1053. def append_backup_stream(self, f_name, stream, basedir=None):
  1054. if not basedir:
  1055. basedir = self.backupdir
  1056. subprocess.check_call(["tar", "-cO", "--posix", "-C", basedir,
  1057. f_name],
  1058. stdout=stream)
  1059. def handle_v3_file(self, f_name, subdir, stream, compressed=True,
  1060. encrypted=True):
  1061. # create inner archive
  1062. tar_cmdline = ["tar", "-Pc", '--sparse',
  1063. '-C', self.fullpath(os.path.dirname(f_name)),
  1064. '--xform', 's:^%s:%s\\0:' % (
  1065. os.path.basename(f_name),
  1066. subdir),
  1067. os.path.basename(f_name)
  1068. ]
  1069. if compressed:
  1070. tar_cmdline.insert(-1, "--use-compress-program=%s" % "gzip")
  1071. tar = subprocess.Popen(tar_cmdline, stdout=subprocess.PIPE)
  1072. if encrypted:
  1073. encryptor = subprocess.Popen(
  1074. ["openssl", "enc", "-e", "-aes-256-cbc", "-pass", "pass:qubes"],
  1075. stdin=tar.stdout,
  1076. stdout=subprocess.PIPE)
  1077. tar.stdout.close()
  1078. data = encryptor.stdout
  1079. else:
  1080. data = tar.stdout
  1081. encryptor = None
  1082. stage1_dir = self.fullpath(os.path.join("stage1", subdir))
  1083. if not os.path.exists(stage1_dir):
  1084. os.makedirs(stage1_dir)
  1085. subprocess.check_call(["split", "--numeric-suffixes",
  1086. "--suffix-length=3",
  1087. "--bytes="+str(100*1024*1024), "-",
  1088. os.path.join(stage1_dir,
  1089. os.path.basename(f_name+"."))],
  1090. stdin=data)
  1091. data.close()
  1092. tar.wait()
  1093. if encryptor:
  1094. encryptor.wait()
  1095. for part in sorted(os.listdir(stage1_dir)):
  1096. if not re.match(
  1097. r"^{}.[0-9][0-9][0-9]$".format(os.path.basename(f_name)),
  1098. part):
  1099. continue
  1100. part_with_dir = os.path.join(subdir, part)
  1101. self.calculate_hmac(os.path.join("stage1", part_with_dir))
  1102. self.append_backup_stream(part_with_dir, stream,
  1103. basedir=self.fullpath("stage1"))
  1104. self.append_backup_stream(part_with_dir+".hmac", stream,
  1105. basedir=self.fullpath("stage1"))
  1106. def handle_v4_file(self, f_name, subdir, stream, compressed="gzip"):
  1107. # create inner archive
  1108. tar_cmdline = ["tar", "-Pc", '--sparse',
  1109. '-C', self.fullpath(os.path.dirname(f_name)),
  1110. '--xform', 's:^%s:%s\\0:' % (
  1111. os.path.basename(f_name),
  1112. subdir),
  1113. os.path.basename(f_name)
  1114. ]
  1115. if compressed and isinstance(compressed, str):
  1116. tar_cmdline.insert(-1, "--use-compress-program=%s" % compressed)
  1117. elif compressed:
  1118. tar_cmdline.insert(-1, "--use-compress-program=%s" % "gzip")
  1119. tar = subprocess.Popen(tar_cmdline, stdout=subprocess.PIPE)
  1120. data = tar.stdout
  1121. stage1_dir = self.fullpath(os.path.join("stage1", subdir))
  1122. if not os.path.exists(stage1_dir):
  1123. os.makedirs(stage1_dir)
  1124. subprocess.check_call(["split", "--numeric-suffixes",
  1125. "--suffix-length=3",
  1126. "--bytes="+str(100*1024*1024), "-",
  1127. os.path.join(stage1_dir,
  1128. os.path.basename(f_name+"."))],
  1129. stdin=data)
  1130. data.close()
  1131. tar.wait()
  1132. for part in sorted(os.listdir(stage1_dir)):
  1133. if not re.match(
  1134. r"^{}.[0-9][0-9][0-9]$".format(os.path.basename(f_name)),
  1135. part):
  1136. continue
  1137. part_with_dir = os.path.join(subdir, part)
  1138. f_name = self.scrypt_encrypt(part_with_dir,
  1139. basedir=self.fullpath('stage1'))
  1140. self.append_backup_stream(f_name, stream,
  1141. basedir=self.fullpath("stage1"))
  1142. def create_v3_backup(self, encrypted=True, compressed=True):
  1143. """
  1144. Create "backup format 3" backup - used in R2 and R3.0
  1145. :param encrypted: Should the backup be encrypted
  1146. :param compressed: Should the backup be compressed
  1147. :return:
  1148. """
  1149. output = open(self.fullpath("backup.bin"), "w")
  1150. with open(self.fullpath("backup-header"), "w") as f:
  1151. f.write(BACKUP_HEADER_R2.format(
  1152. encrypted=str(encrypted),
  1153. compressed=str(compressed)
  1154. ))
  1155. self.calculate_hmac("backup-header")
  1156. self.append_backup_stream("backup-header", output)
  1157. self.append_backup_stream("backup-header.hmac", output)
  1158. with open(self.fullpath("qubes.xml"), "wb") as f:
  1159. qubesxml = pkg_resources.resource_string(__name__, 'v3-qubes.xml')
  1160. if encrypted:
  1161. for vmname, subdir in MANGLED_SUBDIRS_R2.items():
  1162. qubesxml = re.sub(r"[a-z-]*/{}".format(vmname).encode(),
  1163. subdir.encode(), qubesxml)
  1164. f.write(qubesxml)
  1165. else:
  1166. f.write(qubesxml)
  1167. self.handle_v3_file("qubes.xml", "", output, encrypted=encrypted,
  1168. compressed=compressed)
  1169. self.create_v1_files(r2b2=True)
  1170. for vm_type in ["appvms", "servicevms"]:
  1171. for vm_name in os.listdir(self.fullpath(vm_type)):
  1172. vm_dir = os.path.join(vm_type, vm_name)
  1173. for f_name in os.listdir(self.fullpath(vm_dir)):
  1174. if encrypted:
  1175. subdir = MANGLED_SUBDIRS_R2[vm_name]
  1176. else:
  1177. subdir = vm_dir
  1178. self.handle_v3_file(
  1179. os.path.join(vm_dir, f_name),
  1180. subdir+'/', output,
  1181. compressed=compressed,
  1182. encrypted=encrypted)
  1183. for vm_name in os.listdir(self.fullpath("vm-templates")):
  1184. vm_dir = os.path.join("vm-templates", vm_name)
  1185. if encrypted:
  1186. subdir = MANGLED_SUBDIRS_R2[vm_name]
  1187. else:
  1188. subdir = vm_dir
  1189. self.handle_v3_file(
  1190. os.path.join(vm_dir, "."),
  1191. subdir+'/', output,
  1192. compressed=compressed,
  1193. encrypted=encrypted)
  1194. self.handle_v3_file(
  1195. 'dom0-home/user',
  1196. 'dom0-home/', output,
  1197. compressed=compressed,
  1198. encrypted=encrypted)
  1199. output.close()
  1200. def create_v4_backup(self, compressed="gzip"):
  1201. """
  1202. Create "backup format 4" backup - used in R4.0
  1203. :param compressed: Should the backup be compressed
  1204. :return:
  1205. """
  1206. output = open(self.fullpath("backup.bin"), "w")
  1207. with open(self.fullpath("backup-header"), "w") as f:
  1208. f.write(BACKUP_HEADER_R4.format(
  1209. compressed=str(bool(compressed)),
  1210. compression_filter=(compressed if compressed else "gzip")
  1211. ))
  1212. self.scrypt_encrypt("backup-header", output_name='backup-header.hmac')
  1213. self.append_backup_stream("backup-header", output)
  1214. self.append_backup_stream("backup-header.hmac", output)
  1215. with open(self.fullpath("qubes.xml"), "wb") as f:
  1216. qubesxml = pkg_resources.resource_string(__name__, 'v4-qubes.xml')
  1217. for vmname, subdir in MANGLED_SUBDIRS_R4.items():
  1218. qubesxml = re.sub(
  1219. r'backup-path">[a-z-]*/{}'.format(vmname).encode(),
  1220. ('backup-path">' + subdir).encode(),
  1221. qubesxml)
  1222. f.write(qubesxml)
  1223. self.handle_v4_file("qubes.xml", "", output, compressed=compressed)
  1224. self.create_v4_files()
  1225. for vm_type in ["appvms", "vm-templates"]:
  1226. for vm_name in os.listdir(self.fullpath(vm_type)):
  1227. vm_dir = os.path.join(vm_type, vm_name)
  1228. for f_name in os.listdir(self.fullpath(vm_dir)):
  1229. subdir = MANGLED_SUBDIRS_R4[vm_name]
  1230. self.handle_v4_file(
  1231. os.path.join(vm_dir, f_name),
  1232. subdir+'/', output, compressed=compressed)
  1233. self.handle_v4_file(
  1234. 'dom0-home/user',
  1235. 'dom0-home/', output,
  1236. compressed=compressed)
  1237. output.close()
  1238. def setup_expected_calls(self, parsed_qubes_xml, templates_map=None):
  1239. if templates_map is None:
  1240. templates_map = {}
  1241. extra_vm_list_lines = []
  1242. for name, vm in parsed_qubes_xml['domains'].items():
  1243. if not vm['included_in_backup']:
  1244. continue
  1245. if name == 'dom0':
  1246. continue
  1247. if self.storage_pool:
  1248. self.app.expected_calls[
  1249. ('dom0', 'admin.vm.CreateInPool.' + vm['klass'],
  1250. templates_map.get(vm['template'], vm['template']),
  1251. 'name={} label={} pool={}'.format(
  1252. name, vm['label'], self.storage_pool).encode())] = \
  1253. b'0\0'
  1254. else:
  1255. self.app.expected_calls[
  1256. ('dom0', 'admin.vm.Create.' + vm['klass'],
  1257. templates_map.get(vm['template'], vm['template']),
  1258. 'name={} label={}'.format(name, vm['label']).encode())] =\
  1259. b'0\0'
  1260. extra_vm_list_lines.append(
  1261. '{} class={} state=Halted\n'.format(name, vm['klass']).encode())
  1262. if vm['backup_path']:
  1263. self.app.expected_calls[
  1264. (name, 'admin.vm.volume.List', None, None)] = \
  1265. b'0\0root\nprivate\nvolatile\n'
  1266. if vm['klass'] == 'AppVM':
  1267. self.app.expected_calls[
  1268. (name, 'admin.vm.volume.Info', 'root', None)] = \
  1269. b'0\0' \
  1270. b'pool=default\n' \
  1271. b'vid=' + name.encode() + b'/root\n' \
  1272. b'size=1024\n' \
  1273. b'usage=512\n' \
  1274. b'rw=False\n' \
  1275. b'snap_on_start=True\n' \
  1276. b'save_on_stop=False\n' \
  1277. b'source=\n' \
  1278. b'internal=True\n' \
  1279. b'revisions_to_keep=3\n'
  1280. else:
  1281. self.app.expected_calls[
  1282. (name, 'admin.vm.volume.Info', 'root', None)] = \
  1283. b'0\0' \
  1284. b'pool=default\n' \
  1285. b'vid=' + name.encode() + b'/root\n' \
  1286. b'size=1024\n' \
  1287. b'usage=512\n' \
  1288. b'rw=True\n' \
  1289. b'snap_on_start=False\n' \
  1290. b'save_on_stop=True\n' \
  1291. b'internal=True\n' \
  1292. b'revisions_to_keep=3\n'
  1293. self.app.expected_calls[
  1294. (name, 'admin.vm.volume.Resize', 'root',
  1295. str(vm.get('root_size', 2097664)).encode())] = \
  1296. b'0\0'
  1297. self.app.expected_calls[
  1298. (name, 'admin.vm.volume.Import', 'root',
  1299. name.encode() + b'/root')] = \
  1300. b'0\0'
  1301. self.app.expected_calls[
  1302. (name, 'admin.vm.volume.Info', 'private', None)] = \
  1303. b'0\0' \
  1304. b'pool=default\n' \
  1305. b'vid=' + name.encode() + b'/private\n' \
  1306. b'size=1024\n' \
  1307. b'usage=512\n' \
  1308. b'rw=True\n' \
  1309. b'snap_on_start=False\n' \
  1310. b'save_on_stop=True\n' \
  1311. b'revisions_to_keep=3\n'
  1312. self.app.expected_calls[
  1313. (name, 'admin.vm.volume.Resize', 'private', b'2097152')] = \
  1314. b'0\0'
  1315. self.app.expected_calls[
  1316. (name, 'admin.vm.volume.Import', 'private',
  1317. name.encode() + b'/private')] = \
  1318. b'0\0'
  1319. self.app.expected_calls[
  1320. (name, 'admin.vm.volume.Info', 'volatile', None)] = \
  1321. b'0\0' \
  1322. b'pool=default\n' \
  1323. b'vid=' + name.encode() + b'/root\n' \
  1324. b'size=1024\n' \
  1325. b'usage=512\n' \
  1326. b'rw=True\n' \
  1327. b'snap_on_start=False\n' \
  1328. b'save_on_stop=False\n' \
  1329. b'revisions_to_keep=3\n'
  1330. for prop, value in vm['properties'].items():
  1331. self.app.expected_calls[
  1332. (name, 'admin.vm.property.Set', prop,
  1333. str(value).encode() if value is not None else b'')] = b'0\0'
  1334. for bus, devices in vm['devices'].items():
  1335. for (backend_domain, ident), options in devices.items():
  1336. all_options = options.copy()
  1337. all_options['persistent'] = True
  1338. encoded_options = ' '.join('{}={}'.format(key, value) for
  1339. key, value in all_options.items()).encode()
  1340. self.app.expected_calls[
  1341. (name, 'admin.vm.device.{}.Attach'.format(bus),
  1342. '{}+{}'.format(backend_domain, ident),
  1343. encoded_options)] = b'0\0'
  1344. for feature, value in vm['features'].items():
  1345. if value is False:
  1346. value = ''
  1347. elif value is True:
  1348. value = '1'
  1349. self.app.expected_calls[
  1350. (name, 'admin.vm.feature.Set', feature,
  1351. str(value).encode())] = b'0\0'
  1352. for tag in vm['tags']:
  1353. self.app.expected_calls[
  1354. (name, 'admin.vm.tag.Set', tag, None)] = b'0\0'
  1355. if vm['backup_path']:
  1356. appmenus = (
  1357. b'gnome-terminal.desktop\n'
  1358. b'nautilus.desktop\n'
  1359. b'firefox.desktop\n'
  1360. b'mozilla-thunderbird.desktop\n'
  1361. b'libreoffice-startcenter.desktop\n'
  1362. )
  1363. self.app.expected_calls[
  1364. (name, 'appmenus', None, appmenus)] = b'0\0'
  1365. orig_admin_vm_list = self.app.expected_calls[
  1366. ('dom0', 'admin.vm.List', None, None)]
  1367. self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \
  1368. [orig_admin_vm_list] + \
  1369. [orig_admin_vm_list + b''.join(extra_vm_list_lines)] * \
  1370. len(extra_vm_list_lines)
  1371. def mock_appmenus(self, queue, vm, stream):
  1372. queue.put((vm.name, 'appmenus', None, stream.read()))
  1373. def test_210_r2(self):
  1374. self.create_v3_backup(False)
  1375. self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
  1376. b'0\0dom0 class=AdminVM state=Running\n'
  1377. b'fedora-25 class=TemplateVM state=Halted\n'
  1378. b'testvm class=AppVM state=Running\n'
  1379. )
  1380. self.app.expected_calls[
  1381. ('dom0', 'admin.property.Get', 'default_template', None)] = \
  1382. b'0\0default=no type=vm fedora-25'
  1383. self.setup_expected_calls(parsed_qubes_xml_r2, templates_map={
  1384. 'fedora-20-x64': 'fedora-25'
  1385. })
  1386. firewall_data = (
  1387. 'action=accept specialtarget=dns\n'
  1388. 'action=accept proto=icmp\n'
  1389. 'action=accept proto=tcp dstports=22-22\n'
  1390. 'action=accept proto=tcp dstports=9418-9418\n'
  1391. 'action=accept proto=tcp dst4=192.168.0.1/32 dstports=1234-1234\n'
  1392. 'action=accept proto=tcp dsthost=fedorahosted.org dstports=443-443\n'
  1393. 'action=accept proto=tcp dsthost=xenbits.xen.org dstports=80-80\n'
  1394. 'action=drop\n'
  1395. )
  1396. self.app.expected_calls[
  1397. ('test-work', 'admin.vm.firewall.Set', None,
  1398. firewall_data.encode())] = b'0\0'
  1399. self.app.expected_calls[
  1400. ('test-custom-template-appvm', 'admin.vm.firewall.Set', None,
  1401. firewall_data.encode())] = b'0\0'
  1402. qubesd_calls_queue = multiprocessing.Queue()
  1403. dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
  1404. patches = [
  1405. mock.patch('qubesadmin.storage.Volume',
  1406. functools.partial(MockVolume, qubesd_calls_queue)),
  1407. mock.patch(
  1408. 'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
  1409. functools.partial(self.mock_appmenus, qubesd_calls_queue)),
  1410. mock.patch(
  1411. 'qubesadmin.firewall.Firewall',
  1412. functools.partial(MockFirewall, qubesd_calls_queue)),
  1413. mock.patch(
  1414. 'time.strftime',
  1415. return_value=dummy_timestamp)
  1416. ]
  1417. for patch in patches:
  1418. patch.start()
  1419. try:
  1420. self.restore_backup(self.fullpath("backup.bin"), options={
  1421. 'use-default-template': True,
  1422. 'use-default-netvm': True,
  1423. })
  1424. finally:
  1425. for patch in patches:
  1426. patch.stop()
  1427. # retrieve calls from other multiprocess.Process instances
  1428. while not qubesd_calls_queue.empty():
  1429. call_args = qubesd_calls_queue.get()
  1430. self.app.qubesd_call(*call_args)
  1431. qubesd_calls_queue.close()
  1432. self.assertAllCalled()
  1433. self.assertDom0Restored(dummy_timestamp)
  1434. def test_220_r2_encrypted(self):
  1435. self.create_v3_backup(True)
  1436. self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
  1437. b'0\0dom0 class=AdminVM state=Running\n'
  1438. b'fedora-25 class=TemplateVM state=Halted\n'
  1439. b'testvm class=AppVM state=Running\n'
  1440. )
  1441. self.app.expected_calls[
  1442. ('dom0', 'admin.property.Get', 'default_template', None)] = \
  1443. b'0\0default=no type=vm fedora-25'
  1444. self.setup_expected_calls(parsed_qubes_xml_r2, templates_map={
  1445. 'fedora-20-x64': 'fedora-25'
  1446. })
  1447. firewall_data = (
  1448. 'action=accept specialtarget=dns\n'
  1449. 'action=accept proto=icmp\n'
  1450. 'action=accept proto=tcp dstports=22-22\n'
  1451. 'action=accept proto=tcp dstports=9418-9418\n'
  1452. 'action=accept proto=tcp dst4=192.168.0.1/32 dstports=1234-1234\n'
  1453. 'action=accept proto=tcp dsthost=fedorahosted.org dstports=443-443\n'
  1454. 'action=accept proto=tcp dsthost=xenbits.xen.org dstports=80-80\n'
  1455. 'action=drop\n'
  1456. )
  1457. self.app.expected_calls[
  1458. ('test-work', 'admin.vm.firewall.Set', None,
  1459. firewall_data.encode())] = b'0\0'
  1460. self.app.expected_calls[
  1461. ('test-custom-template-appvm', 'admin.vm.firewall.Set', None,
  1462. firewall_data.encode())] = b'0\0'
  1463. qubesd_calls_queue = multiprocessing.Queue()
  1464. dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
  1465. patches = [
  1466. mock.patch('qubesadmin.storage.Volume',
  1467. functools.partial(MockVolume, qubesd_calls_queue)),
  1468. mock.patch(
  1469. 'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
  1470. functools.partial(self.mock_appmenus, qubesd_calls_queue)),
  1471. mock.patch(
  1472. 'qubesadmin.firewall.Firewall',
  1473. functools.partial(MockFirewall, qubesd_calls_queue)),
  1474. mock.patch(
  1475. 'time.strftime',
  1476. return_value=dummy_timestamp)
  1477. ]
  1478. for patch in patches:
  1479. patch.start()
  1480. try:
  1481. self.restore_backup(self.fullpath("backup.bin"), options={
  1482. 'use-default-template': True,
  1483. 'use-default-netvm': True,
  1484. })
  1485. finally:
  1486. for patch in patches:
  1487. patch.stop()
  1488. # retrieve calls from other multiprocess.Process instances
  1489. while not qubesd_calls_queue.empty():
  1490. call_args = qubesd_calls_queue.get()
  1491. self.app.qubesd_call(*call_args)
  1492. qubesd_calls_queue.close()
  1493. self.assertAllCalled()
  1494. self.assertDom0Restored(dummy_timestamp)
  1495. def test_230_r2_uncompressed(self):
  1496. self.create_v3_backup(False, False)
  1497. self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
  1498. b'0\0dom0 class=AdminVM state=Running\n'
  1499. b'fedora-25 class=TemplateVM state=Halted\n'
  1500. b'testvm class=AppVM state=Running\n'
  1501. )
  1502. self.app.expected_calls[
  1503. ('dom0', 'admin.property.Get', 'default_template', None)] = \
  1504. b'0\0default=no type=vm fedora-25'
  1505. self.setup_expected_calls(parsed_qubes_xml_r2, templates_map={
  1506. 'fedora-20-x64': 'fedora-25'
  1507. })
  1508. firewall_data = (
  1509. 'action=accept specialtarget=dns\n'
  1510. 'action=accept proto=icmp\n'
  1511. 'action=accept proto=tcp dstports=22-22\n'
  1512. 'action=accept proto=tcp dstports=9418-9418\n'
  1513. 'action=accept proto=tcp dst4=192.168.0.1/32 dstports=1234-1234\n'
  1514. 'action=accept proto=tcp dsthost=fedorahosted.org dstports=443-443\n'
  1515. 'action=accept proto=tcp dsthost=xenbits.xen.org dstports=80-80\n'
  1516. 'action=drop\n'
  1517. )
  1518. self.app.expected_calls[
  1519. ('test-work', 'admin.vm.firewall.Set', None,
  1520. firewall_data.encode())] = b'0\0'
  1521. self.app.expected_calls[
  1522. ('test-custom-template-appvm', 'admin.vm.firewall.Set', None,
  1523. firewall_data.encode())] = b'0\0'
  1524. qubesd_calls_queue = multiprocessing.Queue()
  1525. dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
  1526. patches = [
  1527. mock.patch('qubesadmin.storage.Volume',
  1528. functools.partial(MockVolume, qubesd_calls_queue)),
  1529. mock.patch(
  1530. 'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
  1531. functools.partial(self.mock_appmenus, qubesd_calls_queue)),
  1532. mock.patch(
  1533. 'qubesadmin.firewall.Firewall',
  1534. functools.partial(MockFirewall, qubesd_calls_queue)),
  1535. mock.patch(
  1536. 'time.strftime',
  1537. return_value=dummy_timestamp)
  1538. ]
  1539. for patch in patches:
  1540. patch.start()
  1541. try:
  1542. self.restore_backup(self.fullpath("backup.bin"), options={
  1543. 'use-default-template': True,
  1544. 'use-default-netvm': True,
  1545. })
  1546. finally:
  1547. for patch in patches:
  1548. patch.stop()
  1549. # retrieve calls from other multiprocess.Process instances
  1550. while not qubesd_calls_queue.empty():
  1551. call_args = qubesd_calls_queue.get()
  1552. self.app.qubesd_call(*call_args)
  1553. qubesd_calls_queue.close()
  1554. self.assertAllCalled()
  1555. self.assertDom0Restored(dummy_timestamp)
  1556. @unittest.skipUnless(spawn.find_executable('scrypt'),
  1557. "scrypt not installed")
  1558. def test_230_r4(self):
  1559. self.create_v4_backup("")
  1560. self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
  1561. b'0\0dom0 class=AdminVM state=Running\n'
  1562. b'fedora-25 class=TemplateVM state=Halted\n'
  1563. b'testvm class=AppVM state=Running\n'
  1564. b'sys-net class=AppVM state=Running\n'
  1565. )
  1566. self.app.expected_calls[
  1567. ('dom0', 'admin.property.Get', 'default_template', None)] = \
  1568. b'0\0default=no type=vm fedora-25'
  1569. self.app.expected_calls[
  1570. ('sys-net', 'admin.vm.property.Get', 'provides_network', None)] = \
  1571. b'0\0default=no type=bool True'
  1572. self.setup_expected_calls(parsed_qubes_xml_v4, templates_map={
  1573. 'debian-8': 'fedora-25'
  1574. })
  1575. firewall_data = (
  1576. 'action=accept specialtarget=dns\n'
  1577. 'action=accept proto=icmp\n'
  1578. 'action=accept proto=tcp dstports=22-22\n'
  1579. 'action=accept proto=tcp dsthost=www.qubes-os.org '
  1580. 'dstports=443-443\n'
  1581. 'action=accept proto=tcp dst4=192.168.0.0/24\n'
  1582. 'action=drop\n'
  1583. )
  1584. self.app.expected_calls[
  1585. ('test-work', 'admin.vm.firewall.Set', None,
  1586. firewall_data.encode())] = b'0\0'
  1587. qubesd_calls_queue = multiprocessing.Queue()
  1588. dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
  1589. patches = [
  1590. mock.patch('qubesadmin.storage.Volume',
  1591. functools.partial(MockVolume, qubesd_calls_queue)),
  1592. mock.patch(
  1593. 'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
  1594. functools.partial(self.mock_appmenus, qubesd_calls_queue)),
  1595. mock.patch(
  1596. 'qubesadmin.firewall.Firewall',
  1597. functools.partial(MockFirewall, qubesd_calls_queue)),
  1598. mock.patch(
  1599. 'time.strftime',
  1600. return_value=dummy_timestamp)
  1601. ]
  1602. for patch in patches:
  1603. patch.start()
  1604. try:
  1605. self.restore_backup(self.fullpath("backup.bin"), options={
  1606. 'use-default-template': True,
  1607. 'use-default-netvm': True,
  1608. })
  1609. finally:
  1610. for patch in patches:
  1611. patch.stop()
  1612. # retrieve calls from other multiprocess.Process instances
  1613. while not qubesd_calls_queue.empty():
  1614. call_args = qubesd_calls_queue.get()
  1615. self.app.qubesd_call(*call_args)
  1616. qubesd_calls_queue.close()
  1617. self.assertAllCalled()
  1618. self.assertDom0Restored(dummy_timestamp)
  1619. @unittest.skipUnless(spawn.find_executable('scrypt'),
  1620. "scrypt not installed")
  1621. def test_230_r4_compressed(self):
  1622. self.create_v4_backup("gzip")
  1623. self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
  1624. b'0\0dom0 class=AdminVM state=Running\n'
  1625. b'fedora-25 class=TemplateVM state=Halted\n'
  1626. b'testvm class=AppVM state=Running\n'
  1627. b'sys-net class=AppVM state=Running\n'
  1628. )
  1629. self.app.expected_calls[
  1630. ('dom0', 'admin.property.Get', 'default_template', None)] = \
  1631. b'0\0default=no type=vm fedora-25'
  1632. self.app.expected_calls[
  1633. ('sys-net', 'admin.vm.property.Get', 'provides_network', None)] = \
  1634. b'0\0default=no type=bool True'
  1635. self.setup_expected_calls(parsed_qubes_xml_v4, templates_map={
  1636. 'debian-8': 'fedora-25'
  1637. })
  1638. firewall_data = (
  1639. 'action=accept specialtarget=dns\n'
  1640. 'action=accept proto=icmp\n'
  1641. 'action=accept proto=tcp dstports=22-22\n'
  1642. 'action=accept proto=tcp dsthost=www.qubes-os.org '
  1643. 'dstports=443-443\n'
  1644. 'action=accept proto=tcp dst4=192.168.0.0/24\n'
  1645. 'action=drop\n'
  1646. )
  1647. self.app.expected_calls[
  1648. ('test-work', 'admin.vm.firewall.Set', None,
  1649. firewall_data.encode())] = b'0\0'
  1650. qubesd_calls_queue = multiprocessing.Queue()
  1651. dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
  1652. patches = [
  1653. mock.patch('qubesadmin.storage.Volume',
  1654. functools.partial(MockVolume, qubesd_calls_queue)),
  1655. mock.patch(
  1656. 'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
  1657. functools.partial(self.mock_appmenus, qubesd_calls_queue)),
  1658. mock.patch(
  1659. 'qubesadmin.firewall.Firewall',
  1660. functools.partial(MockFirewall, qubesd_calls_queue)),
  1661. mock.patch(
  1662. 'time.strftime',
  1663. return_value=dummy_timestamp)
  1664. ]
  1665. for patch in patches:
  1666. patch.start()
  1667. try:
  1668. self.restore_backup(self.fullpath("backup.bin"), options={
  1669. 'use-default-template': True,
  1670. 'use-default-netvm': True,
  1671. })
  1672. finally:
  1673. for patch in patches:
  1674. patch.stop()
  1675. # retrieve calls from other multiprocess.Process instances
  1676. while not qubesd_calls_queue.empty():
  1677. call_args = qubesd_calls_queue.get()
  1678. self.app.qubesd_call(*call_args)
  1679. qubesd_calls_queue.close()
  1680. self.assertAllCalled()
  1681. self.assertDom0Restored(dummy_timestamp)
  1682. @unittest.skipUnless(spawn.find_executable('scrypt'),
  1683. "scrypt not installed")
  1684. def test_230_r4_custom_cmpression(self):
  1685. self.create_v4_backup("bzip2")
  1686. self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
  1687. b'0\0dom0 class=AdminVM state=Running\n'
  1688. b'fedora-25 class=TemplateVM state=Halted\n'
  1689. b'testvm class=AppVM state=Running\n'
  1690. b'sys-net class=AppVM state=Running\n'
  1691. )
  1692. self.app.expected_calls[
  1693. ('dom0', 'admin.property.Get', 'default_template', None)] = \
  1694. b'0\0default=no type=vm fedora-25'
  1695. self.app.expected_calls[
  1696. ('sys-net', 'admin.vm.property.Get', 'provides_network', None)] = \
  1697. b'0\0default=no type=bool True'
  1698. self.setup_expected_calls(parsed_qubes_xml_v4, templates_map={
  1699. 'debian-8': 'fedora-25'
  1700. })
  1701. firewall_data = (
  1702. 'action=accept specialtarget=dns\n'
  1703. 'action=accept proto=icmp\n'
  1704. 'action=accept proto=tcp dstports=22-22\n'
  1705. 'action=accept proto=tcp dsthost=www.qubes-os.org '
  1706. 'dstports=443-443\n'
  1707. 'action=accept proto=tcp dst4=192.168.0.0/24\n'
  1708. 'action=drop\n'
  1709. )
  1710. self.app.expected_calls[
  1711. ('test-work', 'admin.vm.firewall.Set', None,
  1712. firewall_data.encode())] = b'0\0'
  1713. qubesd_calls_queue = multiprocessing.Queue()
  1714. dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
  1715. patches = [
  1716. mock.patch('qubesadmin.storage.Volume',
  1717. functools.partial(MockVolume, qubesd_calls_queue)),
  1718. mock.patch(
  1719. 'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
  1720. functools.partial(self.mock_appmenus, qubesd_calls_queue)),
  1721. mock.patch(
  1722. 'qubesadmin.firewall.Firewall',
  1723. functools.partial(MockFirewall, qubesd_calls_queue)),
  1724. mock.patch(
  1725. 'time.strftime',
  1726. return_value=dummy_timestamp)
  1727. ]
  1728. for patch in patches:
  1729. patch.start()
  1730. try:
  1731. self.restore_backup(self.fullpath("backup.bin"), options={
  1732. 'use-default-template': True,
  1733. 'use-default-netvm': True,
  1734. })
  1735. finally:
  1736. for patch in patches:
  1737. patch.stop()
  1738. # retrieve calls from other multiprocess.Process instances
  1739. while not qubesd_calls_queue.empty():
  1740. call_args = qubesd_calls_queue.get()
  1741. self.app.qubesd_call(*call_args)
  1742. qubesd_calls_queue.close()
  1743. self.assertAllCalled()
  1744. self.assertDom0Restored(dummy_timestamp)
  1745. @unittest.skipUnless(spawn.find_executable('scrypt'),
  1746. "scrypt not installed")
  1747. def test_230_r4_uncommon_compression(self):
  1748. self.create_v4_backup("less")
  1749. qubesd_calls_queue = multiprocessing.Queue()
  1750. dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
  1751. patches = [
  1752. mock.patch('qubesadmin.storage.Volume',
  1753. functools.partial(MockVolume, qubesd_calls_queue)),
  1754. mock.patch(
  1755. 'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
  1756. functools.partial(self.mock_appmenus, qubesd_calls_queue)),
  1757. mock.patch(
  1758. 'qubesadmin.firewall.Firewall',
  1759. functools.partial(MockFirewall, qubesd_calls_queue)),
  1760. mock.patch(
  1761. 'time.strftime',
  1762. return_value=dummy_timestamp)
  1763. ]
  1764. for patch in patches:
  1765. patch.start()
  1766. try:
  1767. with self.assertRaises(qubesadmin.exc.QubesException):
  1768. qubesadmin.backup.restore.BackupRestore(
  1769. self.app, self.fullpath("backup.bin"), None, 'qubes')
  1770. finally:
  1771. for patch in patches:
  1772. patch.stop()
  1773. @unittest.skipUnless(spawn.find_executable('scrypt'),
  1774. "scrypt not installed")
  1775. def test_230_r4_uncommon_cmpression_forced(self):
  1776. self.create_v4_backup("less")
  1777. self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
  1778. b'0\0dom0 class=AdminVM state=Running\n'
  1779. b'fedora-25 class=TemplateVM state=Halted\n'
  1780. b'testvm class=AppVM state=Running\n'
  1781. b'sys-net class=AppVM state=Running\n'
  1782. )
  1783. self.app.expected_calls[
  1784. ('dom0', 'admin.property.Get', 'default_template', None)] = \
  1785. b'0\0default=no type=vm fedora-25'
  1786. self.app.expected_calls[
  1787. ('sys-net', 'admin.vm.property.Get', 'provides_network', None)] = \
  1788. b'0\0default=no type=bool True'
  1789. self.setup_expected_calls(parsed_qubes_xml_v4, templates_map={
  1790. 'debian-8': 'fedora-25'
  1791. })
  1792. firewall_data = (
  1793. 'action=accept specialtarget=dns\n'
  1794. 'action=accept proto=icmp\n'
  1795. 'action=accept proto=tcp dstports=22-22\n'
  1796. 'action=accept proto=tcp dsthost=www.qubes-os.org '
  1797. 'dstports=443-443\n'
  1798. 'action=accept proto=tcp dst4=192.168.0.0/24\n'
  1799. 'action=drop\n'
  1800. )
  1801. self.app.expected_calls[
  1802. ('test-work', 'admin.vm.firewall.Set', None,
  1803. firewall_data.encode())] = b'0\0'
  1804. qubesd_calls_queue = multiprocessing.Queue()
  1805. dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
  1806. patches = [
  1807. mock.patch('qubesadmin.storage.Volume',
  1808. functools.partial(MockVolume, qubesd_calls_queue)),
  1809. mock.patch(
  1810. 'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
  1811. functools.partial(self.mock_appmenus, qubesd_calls_queue)),
  1812. mock.patch(
  1813. 'qubesadmin.firewall.Firewall',
  1814. functools.partial(MockFirewall, qubesd_calls_queue)),
  1815. mock.patch(
  1816. 'time.strftime',
  1817. return_value=dummy_timestamp)
  1818. ]
  1819. for patch in patches:
  1820. patch.start()
  1821. try:
  1822. self.restore_backup(self.fullpath("backup.bin"), options={
  1823. 'use-default-template': True,
  1824. 'use-default-netvm': True,
  1825. }, force_compression_filter='less')
  1826. finally:
  1827. for patch in patches:
  1828. patch.stop()
  1829. # retrieve calls from other multiprocess.Process instances
  1830. while not qubesd_calls_queue.empty():
  1831. call_args = qubesd_calls_queue.get()
  1832. self.app.qubesd_call(*call_args)
  1833. qubesd_calls_queue.close()
  1834. self.assertAllCalled()
  1835. self.assertDom0Restored(dummy_timestamp)
  1836. class TC_11_BackupCompatibilityIntoLVM(TC_10_BackupCompatibility):
  1837. storage_pool = 'some-pool'
  1838. def restore_backup(self, source=None, appvm=None, options=None,
  1839. **kwargs):
  1840. if options is None:
  1841. options = {}
  1842. options['override_pool'] = self.storage_pool
  1843. super(TC_11_BackupCompatibilityIntoLVM, self).restore_backup(source,
  1844. appvm, options, **kwargs)