Merge remote-tracking branch 'origin/pr/145'

* origin/pr/145: (119 commits)
  qvm-template: fix downloading template for install
  tests: add tests for other qvm-template functions
  tests: improve TestProcess behavior
  tests: add tests for qvm-template reinstall/up/downgrade when nothing needs to be done
  tests: fix mock return values of get_dl_list when testing `qvm-template reinstall`
  qvm-template: update comments to reflect e424c7d
  qvm-template: only ask for confirmation during install if something is being done
  tests: add more tests re. install, remove, and get_keys_for_repos
  qvm-template: test != 1 instead of == 0 for template-dummy feature
  tests: fix tests for verify_rpm involving incorrect template names
  tests: add tests for qvm-template remove
  tests: some more for qvm-template
  qvm-template: mute pylint complains about typing.NamedTuple
  tests: qvm-template-postprocess - template.conf handling
  qvm-template-postprocess: fix allowed features list
  qvm-template-postprocess: extract config handling into separate function
  qvm-template-postprocess: treat missing appmenus files as warnings only
  qvm-template: default confirm to 'n'
  qvm-template: verify template package signature directly at download
  qvm-template: improve error reporting
  ...

.pylintrc

@@ -148,6 +148,8 @@ ext-import-graph=
 # not be disabled)
 int-import-graph=
 
+ignored-modules=dnf
+
 
 [DESIGN]
 

ci/requirements.txt

@@ -10,3 +10,5 @@ lxml
 PyYAML
 xcffib
 asynctest
+tqdm
+pyxdg

debian/control

@@ -23,6 +23,7 @@ Package: qubes-core-admin-client
 Architecture: any
 Depends:
  python3-qubesadmin,
+ qubes-repo-templates,
  scrypt,
  ${python:Depends},
  ${python3:Depends},

doc/manpages/qvm-template.rst

@@ -0,0 +1,497 @@
+.. program:: qvm-template
+
+:program:`qvm-template` -- Manage template VMs
+==============================================
+
+Synopsis
+--------
+
+:command:`qvm-template` [-h] [--repo-files *REPO_FILES*] [--keyring *KEYRING*] [--updatevm *UPDATEVM*] [--enablerepo *REPOID*] [--disablerepo *REPOID*] [--repoid *REPOID*] [--releasever *RELEASEVER*] [--refresh] [--cachedir *CACHEDIR*] [--yes] [--quiet] *SUBCOMMAND*
+
+See Section `Subcommands`_ for available subcommands.
+
+Options
+-------
+
+.. option:: --help, -h
+
+   Show help message and exit.
+
+.. option:: --repo-files REPO_FILES
+
+   Specify files containing DNF repository configuration. Can be
+   used more than once. (default:
+   ['/usr/share/qubes/repo-templates/qubes-templates.repo'])
+
+.. option:: --keyring KEYRING
+
+   Specify directory containing RPM public keys. (default:
+   /usr/share/qubes/repo-templates/keys)
+
+.. option:: --updatevm UPDATEVM
+
+   Specify VM to download updates from. (Set to empty string to specify the
+   current VM.) (default: same as UpdateVM - see ``qubes-prefs``)
+
+.. option:: --enablerepo REPOID
+
+   Enable additional repositories by an id or a glob. Can be used more than
+   once.
+
+.. option:: --disablerepo REPOID
+
+   Disable certain repositories by an id or a glob. Can be used more than once.
+
+.. option:: --repoid REPOID
+
+   Enable just specific repositories by an id or a glob. Can be used more than
+   once.
+
+.. option:: --releasever RELEASEVER
+
+   Override Qubes release version.
+
+.. option:: --refresh
+
+   Set repository metadata as expired before running the command.
+
+.. option:: --cachedir CACHEDIR
+
+   Specify cache directory. (default: ~/.cache/qvm-template)
+
+.. option:: --yes
+
+   Assume "yes" to questions.
+
+.. option:: --quiet
+
+   Decrease verbosity.
+
+Subcommands
+===========
+
+install
+-------
+
+Install template packages.
+
+Synopsis
+^^^^^^^^
+
+:command:`qvm-template install` [-h] [--pool *POOL*] [--nogpgcheck] [--allow-pv] [--downloaddir *DOWNLOADDIR*] [--retries *RETRIES*] [*TEMPLATESPEC* [*TEMPLATESPEC* ...]]
+
+See Section `Template Spec`_ for an explanation of *TEMPLATESPEC*.
+
+Options
+^^^^^^^
+
+.. option:: -h, --help
+
+   Show help message and exit.
+
+.. option:: --pool POOL
+
+   Specify pool to store created VMs in.
+
+.. option:: --nogpgcheck
+
+   Disable signature checks.
+
+.. option:: --allow-pv
+
+   Allow templates that set virt_mode to pv.
+
+.. option:: --downloaddir DOWNLOADDIR
+
+   Specify download directory. (default: .)
+
+.. option:: --retries RETRIES
+
+   Specify maximum number of retries for downloads. (default: 5)
+
+{reinstall,downgrade,upgrade}
+-----------------------------
+
+Reinstall/downgrade/upgrade template packages.
+
+Synopsis
+^^^^^^^^
+
+:command:`qvm-template {reinstall,downgrade,upgrade}` [-h] [--nogpgcheck] [--allow-pv] [--downloaddir *DOWNLOADDIR*] [--retries *RETRIES*] [*TEMPLATESPEC* [*TEMPLATESPEC* ...]]
+
+See Section `Template Spec`_ for an explanation of *TEMPLATESPEC*.
+
+Options
+^^^^^^^
+
+.. option:: -h, --help
+
+   Show help message and exit.
+
+.. option:: --nogpgcheck
+
+   Disable signature checks.
+
+.. option:: --allow-pv
+
+   Allow templates that set virt_mode to pv.
+
+.. option:: --downloaddir DOWNLOADDIR
+
+   Specify download directory. (default: .)
+
+.. option:: --retries RETRIES
+
+   Specify maximum number of retries for downloads. (default: 5)
+
+download
+--------
+
+Download template packages.
+
+Synopsis
+^^^^^^^^
+
+:command:`qvm-template download` [-h] [--downloaddir *DOWNLOADDIR*] [--retries *RETRIES*] [*TEMPLATESPEC* [*TEMPLATESPEC* ...]]
+
+See Section `Template Spec`_ for an explanation of *TEMPLATESPEC*.
+
+Options
+^^^^^^^
+
+.. option:: -h, --help
+
+   Show help message and exit.
+
+.. option:: --downloaddir DOWNLOADDIR
+
+   Specify download directory. (default: .)
+
+.. option:: --retries RETRIES
+
+   Specify maximum number of retries for downloads. (default: 5)
+
+list
+----
+
+List templates.
+
+Synopsis
+^^^^^^^^
+
+:command:`qvm-template list` [-h] [--all] [--installed] [--available] [--extras] [--upgrades] [--machine-readable | --machine-readable-json] [*TEMPLATESPEC* [*TEMPLATESPEC* ...]]
+
+See Section `Template Spec`_ for an explanation of *TEMPLATESPEC*.
+
+Options
+^^^^^^^
+
+.. option:: -h, --help
+
+   Show help message and exit.
+
+.. option:: --all
+
+   Show all templates (default).
+
+.. option:: --installed
+
+   Show installed templates.
+
+.. option:: --available
+
+   Show available templates.
+
+.. option:: --extras
+
+   Show extras (e.g., ones that exist locally but not in repos)
+   templates.
+
+.. option:: --upgrades
+
+   Show available upgrades.
+
+.. option:: --machine-readable
+
+   Enable machine-readable output.
+
+   Format
+       Each line describes a template in the following format:
+
+       ::
+
+           {status}|{name}|{evr}|{reponame}
+
+       Where ``{status}`` can be one of ``installed``, ``available``,
+       ``extra``, or ``upgradable``.
+
+       The field ``{evr}`` contains version information in the form of
+       ``{epoch}:{version}-{release}``.
+
+.. option:: --machine-readable-json
+
+   Enable machine-readable output (JSON).
+
+   Format
+       The resulting JSON document is in the following format:
+
+       ::
+
+           {
+               STATUS: [
+                   {
+                       "name": str,
+                       "evr": str,
+                       "reponame": str
+                   },
+                   ...
+               ],
+               ...
+           }
+
+       Where ``STATUS`` can be one of ``"installed"``, ``"available"``,
+       ``"extra"``, or ``"upgradable"``.
+
+       The fields ``buildtime`` and ``installtime`` are in ``%Y-%m-%d
+       %H:%M:%S`` format in UTC.
+
+       The field ``{evr}`` contains version information in the form of
+       ``{epoch}:{version}-{release}``.
+
+info
+----
+
+Display details about templates.
+
+Synopsis
+^^^^^^^^
+
+:command:`qvm-template list` [-h] [--all] [--installed] [--available] [--extras] [--upgrades] [--machine-readable | --machine-readable-json] [*TEMPLATESPEC* [*TEMPLATESPEC* ...]]
+
+See Section `Template Spec`_ for an explanation of *TEMPLATESPEC*.
+
+Options
+^^^^^^^
+
+.. option:: -h, --help
+
+   Show help message and exit.
+
+.. option:: --all
+
+   Show all templates (default).
+
+.. option:: --installed
+
+   Show installed templates.
+
+.. option:: --available
+
+   Show available templates.
+
+.. option:: --extras
+
+   Show extras (e.g., ones that exist locally but not in repos)
+   templates.
+
+.. option:: --upgrades
+
+   Show available upgrades.
+
+.. option:: --machine-readable
+
+   Enable machine-readable output.
+
+   Format
+       Each line describes a template in the following format:
+
+       ::
+
+           {status}|{name}|{epoch}|{version}|{release}|{reponame}|{size}|{buildtime}|{installtime}|{license}|{url}|{summary}|{description}
+
+       Where ``{status}`` can be one of ``installed``, ``available``,
+       ``extra``, or ``upgradable``.
+
+       The fields ``buildtime`` and ``installtime`` are in ``%Y-%m-%d
+       %H:%M:%S`` format in UTC.
+
+       Newlines in the ``{description}`` field are replaced with pipe
+       characters (``|``) for easier processing.
+
+.. option:: --machine-readable-json
+
+   Enable machine-readable output (JSON).
+
+   Format
+       The resulting JSON document is in the following format:
+
+       ::
+
+           {
+               STATUS: [
+                   {
+                       "name": str,
+                       "epoch": str,
+                       "version": str,
+                       "release": str,
+                       "reponame": str,
+                       "size": int,
+                       "buildtime": str,
+                       "installtime": str,
+                       "license": str,
+                       "url": str,
+                       "summary": str,
+                       "description": str
+                   },
+                   ...
+               ],
+               ...
+           }
+
+       Where ``STATUS`` can be one of ``"installed"``, ``"available"``,
+       ``"extra"``, or ``"upgradable"``.
+
+       The fields ``buildtime`` and ``installtime`` are in ``%Y-%m-%d
+       %H:%M:%S`` format in UTC.
+
+search
+------
+
+Search template details for the given string.
+
+Synopsis
+^^^^^^^^
+
+:command:`qvm-template search` [-h] [--all] [*PATTERN* [*PATTERN* ...]]
+
+Options
+^^^^^^^
+
+.. option:: -h, --help
+
+   Show help message and exit.
+
+.. option:: --all
+
+   Search also in the template description and URL. In addition, the criterion
+   are evaluated with OR instead of AND.
+
+remove
+------
+
+Remove installed templates.
+
+Synopsis
+^^^^^^^^
+
+:command:`qvm-template remove` [-h] [--disassoc] [*TEMPLATE* [*TEMPLATE* ...]]
+
+Options
+^^^^^^^
+
+.. option:: -h, --help
+
+   Show help message and exit.
+
+.. option:: --disassoc
+
+   Also disassociate VMs from the templates to be removed. This
+   creates a *dummy* template for the VMs to link with.
+
+purge
+-----
+
+Remove installed templates and associated VMs.
+
+Synopsis
+^^^^^^^^
+
+:command:`qvm-template purge` [-h] [*TEMPLATE* [*TEMPLATE* ...]]
+
+Options
+^^^^^^^
+
+.. option:: -h, --help
+
+   Show help message and exit.
+
+clean
+-----
+
+Remove locally cached packages.
+
+Synopsis
+^^^^^^^^
+
+:command:`qvm-template clean` [-h]
+
+Options
+^^^^^^^
+
+.. option:: -h, --help
+
+   Show help message and exit.
+
+repolist
+--------
+
+Show configured repositories.
+
+Synopsis
+^^^^^^^^
+
+:command:`qvm-template repolist` [-h] [--all | --enabled | --disabled] [*REPOS* [*REPOS* ...]]
+
+Options
+^^^^^^^
+
+.. option:: -h, --help
+
+   Show help message and exit.
+
+.. option:: --all
+
+   Show all repos.
+
+.. option:: --enabled
+
+   Show only enabled repos (default).
+
+.. option:: --disabled
+
+   Show only disabled repos.
+
+Template Spec
+-------------
+
+Subcommands such as ``install`` and ``download`` accept one or more
+*TEMPLATESPEC* strings. The format is, in essence, almost identical to
+``<package-name-spec>`` described in the DNF documentation.
+
+In short, the spec is matched against the following list of NEVRA forms, in
+decreasing orders of priority:
+
+* ``name-[epoch:]version-release``
+* ``name``
+* ``name-[epoch:]version``
+
+Note that unlike DNF, ``arch`` is currently ignored as the template packages
+should all be of ``noarch``.
+
+One can also use globs in spec strings. See Section `Globs`_ for details.
+
+Refer to Section *NEVRA Matching* in the DNF documentation for details.
+
+Globs
+-----
+
+`Template Spec`_ strings, repo ids, and search patterns support glob pattern
+matching. In particular, the following special characters can be used:
+
+* ``*``: Matches any number of characters.
+* ``?``: Matches exactly one character.
+* ``[]``: Matches any enclosed character.
+* ``[!]``: Matches any character except those enclosed.
+
+In particular, note that ``{}``, while supported by DNF, is not supported by
+`qvm-template`.

doc/qubesadmin.tools.rst

@@ -180,6 +180,14 @@ qubesadmin\.tools\.qvm\_tags module
     :undoc-members:
     :show-inheritance:
 
+qubesadmin\.tools\.qvm\_template module
+---------------------------------------
+
+.. automodule:: qubesadmin.tools.qvm_template
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 qubesadmin\.tools\.qvm\_template\_postprocess module
 ----------------------------------------------------
 

qubesadmin/tests/__init__.py

@@ -52,7 +52,7 @@ class TestVMCollection(dict):
 
 
 class TestProcess(object):
-    def __init__(self, input_callback=None, stdout=None, stderr=None):
+    def __init__(self, input_callback=None, stdout=None, stderr=None, stdout_data=None):
         self.input_callback = input_callback
         self.got_any_input = False
         self.stdin = io.BytesIO()
@@ -60,14 +60,20 @@ class TestProcess(object):
         self.stdin_close = self.stdin.close
         self.stdin.close = self.store_input
         self.stdin.flush = self.store_input
-        if stdout == subprocess.PIPE:
+        if stdout == subprocess.PIPE or stdout == subprocess.DEVNULL \
+                or stdout is None:
             self.stdout = io.BytesIO()
         else:
             self.stdout = stdout
-        if stderr == subprocess.PIPE:
+        if stderr == subprocess.PIPE or stderr == subprocess.DEVNULL \
+                or stderr is None:
             self.stderr = io.BytesIO()
         else:
             self.stderr = stderr
+        if stdout_data:
+            self.stdout.write(stdout_data)
+            # Seek to head so that it can be read later
+            self.stdout.seek(0)
         self.returncode = 0
 
     def store_input(self):
@@ -82,14 +88,14 @@ class TestProcess(object):
             self.stdin.write(input)
         self.stdin.close()
         self.stdin_close()
-        return self.stdout, self.stderr
+        return self.stdout.read(), self.stderr.read()
 
     def wait(self):
         self.stdin_close()
         return 0
 
     def poll(self):
-        return None
+        return self.returncode
 
 
 class _AssertNotRaisesContext(object):
@@ -165,11 +171,12 @@ class QubesTest(qubesadmin.app.QubesBase):
         #     raise AssertionError('Unexpected service call {!r}'.format(call_key))
         if call_key in self.expected_service_calls:
             kwargs = kwargs.copy()
-            kwargs['stdout'] = io.BytesIO(self.expected_service_calls[call_key])
+            kwargs['stdout_data'] = self.expected_service_calls[call_key]
         return TestProcess(lambda input: self.service_calls.append((dest,
             service, input)),
             stdout=kwargs.get('stdout', None),
             stderr=kwargs.get('stderr', None),
+            stdout_data=kwargs.get('stdout_data', None),
         )
 
 

qubesadmin/tests/tools/qvm_template_postprocess.py

@@ -17,6 +17,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public License along
 # with this program; if not, see <http://www.gnu.org/licenses/>.
+import argparse
 import asyncio
 import os
 import subprocess
@@ -176,23 +177,51 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase):
         self.assertAllCalled()
 
     def test_010_import_appmenus(self):
+        default_menu_items = [
+            'org.gnome.Terminal.desktop',
+            'firefox.desktop']
+        menu_items = [
+            'org.gnome.Terminal.desktop',
+            'org.gnome.Software.desktop',
+            'gnome-control-center.desktop']
+        netvm_menu_items = [
+            'org.gnome.Terminal.desktop',
+            'nm-connection-editor.desktop']
         with open(os.path.join(self.source_dir.name,
                 'vm-whitelisted-appmenus.list'), 'w') as f:
-            f.write('org.gnome.Terminal.desktop\n')
+            for entry in default_menu_items:
-            f.write('firefox.desktop\n')
+                f.write(entry + '\n')
         with open(os.path.join(self.source_dir.name,
                 'whitelisted-appmenus.list'), 'w') as f:
-            f.write('org.gnome.Terminal.desktop\n')
+            for entry in menu_items:
-            f.write('org.gnome.Software.desktop\n')
+                f.write(entry + '\n')
-            f.write('gnome-control-center.desktop\n')
+        with open(os.path.join(self.source_dir.name,
+                'netvm-whitelisted-appmenus.list'), 'w') as f:
+            for entry in netvm_menu_items:
+                f.write(entry + '\n')
 
         self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \
             b'0\0test-vm class=TemplateVM state=Halted\n'
+        self.app.expected_calls[(
+            'test-vm',
+            'admin.vm.feature.Set',
+            'default-menu-items',
+            ' '.join(default_menu_items).encode())] = b'0\0'
+        self.app.expected_calls[(
+            'test-vm',
+            'admin.vm.feature.Set',
+            'menu-items',
+            ' '.join(menu_items).encode())] = b'0\0'
+        self.app.expected_calls[(
+            'test-vm',
+            'admin.vm.feature.Set',
+            'netvm-menu-items',
+            ' '.join(netvm_menu_items).encode())] = b'0\0'
 
         vm = self.app.domains['test-vm']
         with mock.patch('subprocess.check_call') as mock_proc:
             qubesadmin.tools.qvm_template_postprocess.import_appmenus(
-                vm, self.source_dir.name)
+                vm, self.source_dir.name, skip_generate=False)
         self.assertEqual(mock_proc.mock_calls, [
             mock.call(['qvm-appmenus',
                 '--set-default-whitelist=' + os.path.join(self.source_dir.name,
@@ -205,15 +234,43 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase):
     @mock.patch('grp.getgrnam')
     @mock.patch('os.getuid')
     def test_011_import_appmenus_as_root(self, mock_getuid, mock_getgrnam):
+        default_menu_items = [
+            'org.gnome.Terminal.desktop',
+            'firefox.desktop']
+        menu_items = [
+            'org.gnome.Terminal.desktop',
+            'org.gnome.Software.desktop',
+            'gnome-control-center.desktop']
+        netvm_menu_items = [
+            'org.gnome.Terminal.desktop',
+            'nm-connection-editor.desktop']
         with open(os.path.join(self.source_dir.name,
                 'vm-whitelisted-appmenus.list'), 'w') as f:
-            f.write('org.gnome.Terminal.desktop\n')
+            for entry in default_menu_items:
-            f.write('firefox.desktop\n')
+                f.write(entry + '\n')
         with open(os.path.join(self.source_dir.name,
                 'whitelisted-appmenus.list'), 'w') as f:
-            f.write('org.gnome.Terminal.desktop\n')
+            for entry in menu_items:
-            f.write('org.gnome.Software.desktop\n')
+                f.write(entry + '\n')
-            f.write('gnome-control-center.desktop\n')
+        with open(os.path.join(self.source_dir.name,
+                'netvm-whitelisted-appmenus.list'), 'w') as f:
+            for entry in netvm_menu_items:
+                f.write(entry + '\n')
+        self.app.expected_calls[(
+            'test-vm',
+            'admin.vm.feature.Set',
+            'default-menu-items',
+            ' '.join(default_menu_items).encode())] = b'0\0'
+        self.app.expected_calls[(
+            'test-vm',
+            'admin.vm.feature.Set',
+            'menu-items',
+            ' '.join(menu_items).encode())] = b'0\0'
+        self.app.expected_calls[(
+            'test-vm',
+            'admin.vm.feature.Set',
+            'netvm-menu-items',
+            ' '.join(netvm_menu_items).encode())] = b'0\0'
 
         self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \
             b'0\0test-vm class=TemplateVM state=Halted\n'
@@ -226,7 +283,7 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase):
         vm = self.app.domains['test-vm']
         with mock.patch('subprocess.check_call') as mock_proc:
             qubesadmin.tools.qvm_template_postprocess.import_appmenus(
-                vm, self.source_dir.name)
+                vm, self.source_dir.name, skip_generate=False)
         self.assertEqual(mock_proc.mock_calls, [
             mock.call(['runuser', '-u', 'user', '--', 'env', 'DISPLAY=:0',
                 'qvm-appmenus',
@@ -260,7 +317,7 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase):
         vm = self.app.domains['test-vm']
         with mock.patch('subprocess.check_call') as mock_proc:
             qubesadmin.tools.qvm_template_postprocess.import_appmenus(
-                vm, self.source_dir.name)
+                vm, self.source_dir.name, skip_generate=False)
         self.assertEqual(mock_proc.mock_calls, [])
         self.assertAllCalled()
 
@@ -313,11 +370,11 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase):
             app=self.app)
         self.assertEqual(ret, 0)
         self.app.add_new_vm.assert_called_once_with('TemplateVM',
-            name='test-vm', label='black')
+            name='test-vm', label='black', pool=None)
         mock_import_root_img.assert_called_once_with(self.app.domains[
             'test-vm'], self.source_dir.name)
         mock_import_appmenus.assert_called_once_with(self.app.domains[
-            'test-vm'], self.source_dir.name)
+            'test-vm'], self.source_dir.name, skip_generate=True)
         if qubesadmin.tools.qvm_template_postprocess.have_events:
             mock_domain_shutdown.assert_called_once_with([self.app.domains[
                 'test-vm']])
@@ -372,7 +429,7 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase):
         mock_reset_private_img.assert_called_once_with(self.app.domains[
             'test-vm'])
         mock_import_appmenus.assert_called_once_with(self.app.domains[
-            'test-vm'], self.source_dir.name)
+            'test-vm'], self.source_dir.name, skip_generate=True)
         if qubesadmin.tools.qvm_template_postprocess.have_events:
             mock_domain_shutdown.assert_called_once_with([self.app.domains[
                 'test-vm']])
@@ -413,7 +470,7 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase):
         mock_reset_private_img.assert_called_once_with(self.app.domains[
             'test-vm'])
         mock_import_appmenus.assert_called_once_with(self.app.domains[
-            'test-vm'], self.source_dir.name)
+            'test-vm'], self.source_dir.name, skip_generate=False)
         if qubesadmin.tools.qvm_template_postprocess.have_events:
             self.assertFalse(mock_domain_shutdown.called)
         self.assertEqual(self.app.service_calls, [])
@@ -466,3 +523,119 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase):
                 'post-install', 'test-vm', self.source_dir.name],
                 app=self.app)
         self.assertAllCalled()
+
+    def test_050_template_config(self):
+        template_config = """gui=1
+qrexec=1
+linux-stubdom=1
+net.fake-ip=192.168.1.100
+net.fake-netmask=255.255.255.0
+net.fake-gateway=192.168.1.1
+virt-mode=hvm
+kernel=
+"""
+        template_conf = os.path.join(self.source_dir.name, 'template.conf')
+        with open(template_conf, 'w') as f:
+            f.write(template_config)
+        self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \
+            b'0\0test-vm class=TemplateVM state=Halted\n'
+        self.app.expected_calls[(
+            'test-vm', 'admin.vm.feature.Set', 'gui', b'1')] = b'0\0'
+        self.app.expected_calls[(
+            'test-vm', 'admin.vm.feature.Set', 'qrexec', b'1')] = b'0\0'
+        self.app.expected_calls[(
+            'test-vm', 'admin.vm.feature.Set', 'linux-stubdom', b'1')] = b'0\0'
+        self.app.expected_calls[(
+            'test-vm', 'admin.vm.feature.Set', 'net.fake-ip', b'192.168.1.100')] = b'0\0'
+        self.app.expected_calls[(
+            'test-vm', 'admin.vm.feature.Set', 'net.fake-netmask', b'255.255.255.0')] = b'0\0'
+        self.app.expected_calls[(
+            'test-vm', 'admin.vm.feature.Set', 'net.fake-gateway', b'192.168.1.1')] = b'0\0'
+        self.app.expected_calls[(
+            'test-vm', 'admin.vm.property.Set', 'virt_mode', b'hvm')] = b'0\0'
+        self.app.expected_calls[(
+            'test-vm', 'admin.vm.property.Set', 'kernel', b'')] = b'0\0'
+
+        vm = self.app.domains['test-vm']
+        args = argparse.Namespace(
+            allow_pv=False,
+        )
+        qubesadmin.tools.qvm_template_postprocess.import_template_config(
+            args, template_conf, vm)
+        self.assertAllCalled()
+
+    def test_051_template_config_invalid(self):
+        self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \
+            b'0\0test-vm class=TemplateVM state=Halted\n'
+        vm = self.app.domains['test-vm']
+        args = argparse.Namespace(
+            allow_pv=False,
+        )
+        with self.subTest('invalid feature value'):
+            template_config = "gui=false\n"
+            template_conf = os.path.join(self.source_dir.name, 'template.conf')
+            with open(template_conf, 'w') as f:
+                f.write(template_config)
+            qubesadmin.tools.qvm_template_postprocess.import_template_config(
+                args, template_conf, vm)
+
+        with self.subTest('invalid feature name'):
+            template_config = "invalid=1\n"
+            template_conf = os.path.join(self.source_dir.name, 'template.conf')
+            with open(template_conf, 'w') as f:
+                f.write(template_config)
+            qubesadmin.tools.qvm_template_postprocess.import_template_config(
+                args, template_conf, vm)
+
+        with self.subTest('invalid ip'):
+            template_config = "net.fake-ip=1.2.3.4.5\n"
+            template_conf = os.path.join(self.source_dir.name, 'template.conf')
+            with open(template_conf, 'w') as f:
+                f.write(template_config)
+            qubesadmin.tools.qvm_template_postprocess.import_template_config(
+                args, template_conf, vm)
+
+        with self.subTest('invalid virt-mode'):
+            template_config = "virt-mode=invalid\n"
+            template_conf = os.path.join(self.source_dir.name, 'template.conf')
+            with open(template_conf, 'w') as f:
+                f.write(template_config)
+            qubesadmin.tools.qvm_template_postprocess.import_template_config(
+                args, template_conf, vm)
+
+        with self.subTest('invalid kernel'):
+            template_config = "kernel=1.2.3.4.5\n"
+            template_conf = os.path.join(self.source_dir.name, 'template.conf')
+            with open(template_conf, 'w') as f:
+                f.write(template_config)
+            qubesadmin.tools.qvm_template_postprocess.import_template_config(
+                args, template_conf, vm)
+        self.assertAllCalled()
+
+    def test_052_template_config_virt_mode_pv(self):
+        self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \
+            b'0\0test-vm class=TemplateVM state=Halted\n'
+        vm = self.app.domains['test-vm']
+        args = argparse.Namespace(
+            allow_pv=False,
+        )
+        with self.subTest('not allowed'):
+            template_config = "virt-mode=pv\n"
+            template_conf = os.path.join(self.source_dir.name, 'template.conf')
+            with open(template_conf, 'w') as f:
+                f.write(template_config)
+            qubesadmin.tools.qvm_template_postprocess.import_template_config(
+                args, template_conf, vm)
+        with self.subTest('allowed'):
+            args = argparse.Namespace(
+                allow_pv=True,
+            )
+            self.app.expected_calls[(
+                'test-vm', 'admin.vm.property.Set', 'virt_mode', b'pv')] = b'0\0'
+            template_config = "virt-mode=pv\n"
+            template_conf = os.path.join(self.source_dir.name, 'template.conf')
+            with open(template_conf, 'w') as f:
+                f.write(template_config)
+            qubesadmin.tools.qvm_template_postprocess.import_template_config(
+                args, template_conf, vm)
+        self.assertAllCalled()

qubesadmin/tools/qvm_template_postprocess.py

@@ -23,6 +23,7 @@
 import asyncio
 import glob
 import os
+import pathlib
 
 import shutil
 import subprocess
@@ -49,6 +50,12 @@ parser.add_argument('--skip-start', action='store_true',
     help='Do not start the VM - do not retrieve menu entries etc.')
 parser.add_argument('--keep-source', action='store_true',
     help='Do not remove source data (*dir* directory) after import')
+parser.add_argument('--no-installed-by-rpm', action='store_true',
+    help='Do not set installed_by_rpm')
+parser.add_argument('--allow-pv', action='store_true',
+    help='Allow setting virt_mode to pv in configuration file.')
+parser.add_argument('--pool',
+    help='Specify pool to store created VMs in.')
 parser.add_argument('action', choices=['post-install', 'pre-remove'],
     help='Action to perform')
 parser.add_argument('name', action='store',
@@ -60,9 +67,13 @@ parser.add_argument('dir', action='store',
 def get_root_img_size(source_dir):
     '''Extract size of root.img to be imported'''
     root_path = os.path.join(source_dir, 'root.img')
-    if os.path.exists(root_path + '.part.00'):
+    # deal with both cases: split tar and non-split tar
+    part_path = root_path + '.part.00'
+    tar_path = root_path + '.tar'
+    if os.path.exists(part_path) or os.path.exists(tar_path):
         # get just file root_size from the tar header
-        p = subprocess.Popen(['tar', 'tvf', root_path + '.part.00'],
+        path = part_path if os.path.exists(part_path) else tar_path
+        p = subprocess.Popen(['tar', 'tvf', path],
             stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
         (stdout, _) = p.communicate()
         # -rw-r--r-- 0/0      1073741824 1970-01-01 01:00 root.img
@@ -98,6 +109,12 @@ def import_root_img(vm, source_dir):
             raise qubesadmin.exc.QubesException('root.img extraction failed')
         if cat.wait() != 0:
             raise qubesadmin.exc.QubesException('root.img extraction failed')
+    elif os.path.exists(root_path + '.tar'):
+        tar = subprocess.Popen(['tar', 'xSOf', root_path + '.tar'],
+            stdout=subprocess.PIPE)
+        vm.volumes['root'].import_data(stream=tar.stdout)
+        if tar.wait() != 0:
+            raise qubesadmin.exc.QubesException('root.img extraction failed')
     elif os.path.exists(root_path):
         if vm.app.qubesd_connection_type == 'socket':
             # check if root.img was already overwritten, i.e. if the source
@@ -127,8 +144,14 @@ def reset_private_img(vm):
     vm.volumes['private'].clear_data()
 
 
-def import_appmenus(vm, source_dir):
+def import_appmenus(vm, source_dir, skip_generate=True):
-    '''Import appmenus settings into VM object (later: GUI VM)'''
+    """Import appmenus settings into VM object (later: GUI VM)
+
+    :param vm: QubesVM object of just imported template
+    :param source_dir: directory with source files
+    :param skip_generate: do not generate actual menu entries,
+            only set item lists
+    """
     if os.getuid() == 0:
         try:
             qubes_group = grp.getgrnam('qubes')
@@ -141,18 +164,50 @@ def import_appmenus(vm, source_dir):
     else:
         cmd_prefix = []
 
+    # store the whitelists in VM features
+    # separated by spaces should be ok as there should be no spaces in the file
+    # name according to the FreeDesktop spec
+    source_dir = pathlib.Path(source_dir)
+    try:
+        with open(source_dir / 'vm-whitelisted-appmenus.list', 'r') as fd:
+            vm.features['default-menu-items'] = \
+                ' '.join([x.rstrip() for x in fd])
+    except FileNotFoundError as e:
+        vm.log.warning('Cannot set default-menu-items, %s not found',
+                       e.filename)
+    try:
+        with open(source_dir / 'whitelisted-appmenus.list', 'r') as fd:
+            vm.features['menu-items'] = ' '.join([x.rstrip() for x in fd])
+    except FileNotFoundError as e:
+        vm.log.warning('Cannot set menu-items, %s not found',
+                       e.filename)
+    try:
+        with open(source_dir / 'netvm-whitelisted-appmenus.list', 'r') as fd:
+            vm.features['netvm-menu-items'] = ' '.join([x.rstrip() for x in fd])
+    except FileNotFoundError as e:
+        vm.log.warning('Cannot set netvm-menu-items, %s not found',
+                       e.filename)
+
+    if skip_generate:
+        return
+
     # TODO: change this to qrexec calls to GUI VM, when GUI VM will be
     # implemented
     try:
         subprocess.check_call(cmd_prefix + ['qvm-appmenus',
-            '--set-default-whitelist={}'.format(os.path.join(source_dir,
+            '--set-default-whitelist={!s}'.format(
-                'vm-whitelisted-appmenus.list')), vm.name])
+                source_dir / 'vm-whitelisted-appmenus.list'), vm.name])
         subprocess.check_call(cmd_prefix + ['qvm-appmenus',
-            '--set-whitelist={}'.format(os.path.join(source_dir,
+            '--set-whitelist={!s}'.format(
-                'whitelisted-appmenus.list')), vm.name])
+                source_dir / 'whitelisted-appmenus.list'), vm.name])
     except subprocess.CalledProcessError as e:
         vm.log.warning('Failed to set default application list: %s', e)
 
+def parse_template_config(path):
+    '''Parse template.conf from template package. (KEY=VALUE format)'''
+    with open(path, 'r') as fd:
+        return dict(line.rstrip('\n').split('=', 1) for line in fd)
+
 @asyncio.coroutine
 def call_postinstall_service(vm):
     '''Call qubes.PostInstall service
@@ -197,6 +252,12 @@ def call_postinstall_service(vm):
     finally:
         vm.netvm = qubesadmin.DEFAULT
 
+def validate_ip(ip):
+    """Check if given string has a valid IP address syntax"""
+    try:
+        return all(0 <= int(part) <= 255 for part in ip.split('.', 3))
+    except ValueError:
+        return False
 
 @asyncio.coroutine
 def post_install(args):
@@ -226,7 +287,8 @@ def post_install(args):
 
         vm = app.add_new_vm('TemplateVM',
             name=args.name,
-            label=qubesadmin.config.defaults['template_label'])
+            label=qubesadmin.config.defaults['template_label'],
+            pool=args.pool)
         vm_created = True
 
     vm.log.info('Importing data')
@@ -240,8 +302,14 @@ def post_install(args):
     if not vm_created:
         vm.log.info('Clearing private volume')
         reset_private_img(vm)
-    vm.installed_by_rpm = True
+    vm.installed_by_rpm = not args.no_installed_by_rpm
-    import_appmenus(vm, args.dir)
+    # do not generate actual menu entries, if post-install service will be
+    # executed anyway
+    import_appmenus(vm, args.dir, skip_generate=not args.skip_start)
+
+    conf_path = os.path.join(args.dir, 'template.conf')
+    if os.path.exists(conf_path):
+        import_template_config(args, conf_path, vm)
 
     if not args.skip_start:
         yield from call_postinstall_service(vm)
@@ -263,6 +331,60 @@ def post_install(args):
     return 0
 
 
+def import_template_config(args, conf_path, vm):
+    """
+    Parse template.conf and apply its content to the just installed TemplateVM
+
+    :param args: arguments for qvm-template-postprocess (used for --allow-pv
+        option and possibly some other in the future)
+    :param conf_path: path to the template.conf
+    :param vm: Template to operate on
+    :return:
+    """
+    conf = parse_template_config(conf_path)
+    # Import qvm-feature tags
+    for key in (
+            'no-monitor-layout',
+            'pci-e820-host',
+            'linux-stubdom',
+            'gui',
+            'gui-emulated',
+            'qrexec'):
+        if key in conf:
+            if conf[key] == '1':
+                vm.features[key] = conf[key]
+            else:
+                vm.log.warning(
+                    'ignoring boolean config flags that are not \'1\'')
+    for key in (
+            'net.fake-ip',
+            'net.fake-gateway',
+            'net.fake-netmask'):
+        if key in conf:
+            if validate_ip(conf[key]):
+                vm.features[key] = conf[key]
+            else:
+                vm.log.warning(
+                    'ignoring invalid value for \'%s\'', key)
+    if 'virt-mode' in conf:
+        if conf['virt-mode'] == 'pv' and args.allow_pv:
+            vm.virt_mode = 'pv'
+        elif conf['virt-mode'] == 'pv':
+            vm.log.warning(
+                '--allow-pv not set, ignoring request to change virt-mode')
+        elif conf['virt-mode'] in ('pvh', 'hvm'):
+            vm.virt_mode = conf['virt-mode']
+        else:
+            vm.log.warning('ignoring invalid value for virt-mode')
+
+    if 'kernel' in conf:
+        if conf['kernel'] == '':
+            vm.kernel = ''
+        else:
+            vm.log.warning(
+                'Currently only supports setting kernel to (none)')
+
+
 def pre_remove(args):
     '''Handle pre-removal tasks'''
     app = args.app
@@ -287,7 +409,6 @@ def is_chroot():
             stat_root.st_dev != stat_init_root.st_dev or
             stat_root.st_ino != stat_init_root.st_ino)
     except IOError:
-        print('Stat failed, assuming not chroot', file=sys.stderr)
         return False
 
 

rpm_spec/qubes-core-admin-client.spec.in

@@ -15,6 +15,7 @@ BuildRequires:	python%{python3_pkgversion}-lxml
 BuildRequires:	python%{python3_pkgversion}-xcffib
 Requires:   python%{python3_pkgversion}-qubesadmin
 Requires:   python%{python3_pkgversion}-yaml
+Requires:   qubes-repo-templates
 Requires:   scrypt
 BuildArch:  noarch
 Source0: %{name}-%{version}.tar.gz

test-packages/rpm.py

@@ -0,0 +1,46 @@
+# RPM header tags
+# Generated with the following command:
+# ``grep -Po '(RPMTAG[A-Z_]*)' tools/qvm_template.py | sort | uniq``
+RPMTAG_BUILDTIME   = 1
+RPMTAG_DESCRIPTION = 2
+RPMTAG_EPOCHNUM    = 3
+RPMTAG_LICENSE     = 4
+RPMTAG_NAME        = 5
+RPMTAG_RELEASE     = 6
+RPMTAG_SIGGPG      = 7
+RPMTAG_SIGPGP      = 8
+RPMTAG_SUMMARY     = 9
+RPMTAG_URL         = 10
+RPMTAG_VERSION     = 11
+
+RPMVSF_MASK_NOSIGNATURES = 0xc0c00
+
+class error(BaseException):
+    def __init__(self, msg):
+        self.msg = msg
+
+    def __str__(self):
+        return self.msg
+
+class hdr():
+    def __getitem__(self, key):
+        pass
+
+class keyring():
+    def addKey(self, *args):
+        pass
+
+class pubkey():
+    pass
+
+class TransactionSet():
+    def setVSFlags(self, flags):
+        pass
+    def setKeyring(self, *args):
+        pass
+    def hdrFromFdno(self, fdno) -> hdr:
+        return hdr()
+
+def labelCompare(a, b):
+    # Pretend that we're comparing the versions lexographically in the stub
+    return (a > b) - (a < b)