From 9d9ee6a4b7fac1a6175bae44a27f4681f679bfa6 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 28 Jun 2020 03:07:08 +0800 Subject: [PATCH 001/119] Initial support for qvm-template. --- qubesadmin/tools/qvm_template_postprocess.py | 65 +++++++++++++++++++- 1 file changed, 62 insertions(+), 3 deletions(-) mode change 100644 => 100755 qubesadmin/tools/qvm_template_postprocess.py diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py old mode 100644 new mode 100755 index ca5ec7f..48860c1 --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -49,6 +49,10 @@ parser.add_argument('--skip-start', action='store_true', help='Do not start the VM - do not retrieve menu entries etc.') parser.add_argument('--keep-source', action='store_true', help='Do not remove source data (*dir* directory) after import') +parser.add_argument('--no-installed-by-rpm', action='store_true', + help='Do not set installed_by_rpm') +parser.add_argument('--allow-pv', action='store_true', + help='Allow setting virt_mode to pv in configuration file.') parser.add_argument('action', choices=['post-install', 'pre-remove'], help='Action to perform') parser.add_argument('name', action='store', @@ -60,9 +64,13 @@ parser.add_argument('dir', action='store', def get_root_img_size(source_dir): '''Extract size of root.img to be imported''' root_path = os.path.join(source_dir, 'root.img') - if os.path.exists(root_path + '.part.00'): + # deal with both cases: split tar and non-split tar + part_path = root_path + '.part.00' + tar_path = root_path + '.tar' + if os.path.exists(part_path) or os.path.exists(tar_path): # get just file root_size from the tar header - p = subprocess.Popen(['tar', 'tvf', root_path + '.part.00'], + path = part_path if os.path.exists(part_path) else tar_path + p = subprocess.Popen(['tar', 'tvf', path], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) (stdout, _) = p.communicate() # -rw-r--r-- 0/0 1073741824 1970-01-01 01:00 root.img @@ -98,6 +106,12 @@ def import_root_img(vm, source_dir): raise qubesadmin.exc.QubesException('root.img extraction failed') if cat.wait() != 0: raise qubesadmin.exc.QubesException('root.img extraction failed') + elif os.path.exists(root_path + '.tar'): + tar = subprocess.Popen(['tar', 'xSOf', root_path + '.tar'], + stdout=subprocess.PIPE) + vm.volumes['root'].import_data(stream=tar.stdout) + if tar.wait() != 0: + raise qubesadmin.exc.QubesException('root.img extraction failed') elif os.path.exists(root_path): if vm.app.qubesd_connection_type == 'socket': # check if root.img was already overwritten, i.e. if the source @@ -141,6 +155,17 @@ def import_appmenus(vm, source_dir): else: cmd_prefix = [] + with open(os.path.join(source_dir, 'vm-whitelisted-appmenus.list'), 'r') \ + as f: + vm.features['default-whitelist'] = f.read() + with open(os.path.join(source_dir, 'whitelisted-appmenus.list'), 'r') \ + as f: + vm.features['whitelist'] = f.read() + with open( + os.path.join(source_dir, 'netvm-whitelisted-appmenus.list'), 'r') \ + as f: + vm.features['netvm-whitelist'] = f.read() + # TODO: change this to qrexec calls to GUI VM, when GUI VM will be # implemented try: @@ -153,6 +178,11 @@ def import_appmenus(vm, source_dir): except subprocess.CalledProcessError as e: vm.log.warning('Failed to set default application list: %s', e) +def parse_template_config(path): + '''Parse template.conf from template package. (KEY=VALUE format)''' + with open(path, 'r') as f: + return dict(line.rstrip('\n').split('=', 1) for line in f) + @asyncio.coroutine def call_postinstall_service(vm): '''Call qubes.PostInstall service @@ -240,9 +270,38 @@ def post_install(args): if not vm_created: vm.log.info('Clearing private volume') reset_private_img(vm) - vm.installed_by_rpm = True + vm.installed_by_rpm = not args.no_installed_by_rpm import_appmenus(vm, args.dir) + conf_path = os.path.join(args.dir, 'template.conf') + if os.path.exists(conf_path): + conf = parse_template_config(conf_path) + # Import qvm-feature tags + for key in ( + 'no-monitor-layout', + 'net.fake-ip', + 'net.fake-gateway', + 'net.fake-netmask', + 'pci-e820-host', + 'linux-stubdom', + 'gui', + 'gui-emulated' + 'qrexec'): + if key in conf: + vm.features[key] = conf[key] + if 'virt-mode' in conf: + if conf['virt-mode'] == 'pv' and args.allow_pv: + vm.virt_mode = 'pv' + else: + vm.log.warning( + '--allow-pv not set, ignoring request to change virt-mode') + if 'kernel' in conf: + if conf['kernel'] == '': + vm.kernel = '' + else: + vm.log.warning( + 'Currently only supports setting kernel to (none)') + if not args.skip_start: yield from call_postinstall_service(vm) From 6c7360f25c1f3fffec8369ab8d47168534f45954 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 30 Jun 2020 12:08:16 +0800 Subject: [PATCH 002/119] Separate whitelist entries with spaces instead of newlines. --- qubesadmin/tools/qvm_template_postprocess.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py index 48860c1..4cc5ca4 100755 --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -155,16 +155,19 @@ def import_appmenus(vm, source_dir): else: cmd_prefix = [] + # store the whitelists in VM features + # separated by spaces should be ok as there should be no spaces in the file + # name according to the FreeDesktop spec with open(os.path.join(source_dir, 'vm-whitelisted-appmenus.list'), 'r') \ as f: - vm.features['default-whitelist'] = f.read() + vm.features['default-whitelist'] = ' '.join([x.rstrip() for x in f]) with open(os.path.join(source_dir, 'whitelisted-appmenus.list'), 'r') \ as f: - vm.features['whitelist'] = f.read() + vm.features['whitelist'] = ' '.join([x.rstrip() for x in f]) with open( os.path.join(source_dir, 'netvm-whitelisted-appmenus.list'), 'r') \ as f: - vm.features['netvm-whitelist'] = f.read() + vm.features['netvm-whitelist'] = ' '.join([x.rstrip() for x in f]) # TODO: change this to qrexec calls to GUI VM, when GUI VM will be # implemented From eda68cce6ded7681e0d0e3e9fd3c01a6830c35ab Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 30 Jun 2020 12:09:22 +0800 Subject: [PATCH 003/119] Verify values of boolean flags in template config. --- qubesadmin/tools/qvm_template_postprocess.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py index 4cc5ca4..cdc40c3 100755 --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -282,14 +282,21 @@ def post_install(args): # Import qvm-feature tags for key in ( 'no-monitor-layout', - 'net.fake-ip', - 'net.fake-gateway', - 'net.fake-netmask', 'pci-e820-host', 'linux-stubdom', 'gui', 'gui-emulated' 'qrexec'): + if key in conf: + if conf[key] == '1': + vm.features[key] = conf[key] + else: + vm.log.warning( + 'ignoring boolean config flags that are not \'1\'') + for key in ( + 'net.fake-ip', + 'net.fake-gateway', + 'net.fake-netmask'): if key in conf: vm.features[key] = conf[key] if 'virt-mode' in conf: From e8ba117c2633c7aa0a505128f174a6698e40eb37 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 30 Jun 2020 12:14:50 +0800 Subject: [PATCH 004/119] Allow virt_mode other than pv. --- qubesadmin/tools/qvm_template_postprocess.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py index cdc40c3..7351266 100755 --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -302,9 +302,12 @@ def post_install(args): if 'virt-mode' in conf: if conf['virt-mode'] == 'pv' and args.allow_pv: vm.virt_mode = 'pv' - else: + elif conf['virt-mode'] == 'pv': vm.log.warning( '--allow-pv not set, ignoring request to change virt-mode') + elif conf['virt-mode'] in ('pvh', 'hvm'): + vm.virt_mode = conf['virt-mode'] + if 'kernel' in conf: if conf['kernel'] == '': vm.kernel = '' From bab8e699d72d02850a03bd9d5618689da061dc15 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Mon, 6 Jul 2020 01:16:43 +0800 Subject: [PATCH 005/119] Change "whitelist" to "menu-items" in qvm-features for clarity. --- qubesadmin/tools/qvm_template_postprocess.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100755 => 100644 qubesadmin/tools/qvm_template_postprocess.py diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py old mode 100755 new mode 100644 index 7351266..568cfde --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -160,14 +160,14 @@ def import_appmenus(vm, source_dir): # name according to the FreeDesktop spec with open(os.path.join(source_dir, 'vm-whitelisted-appmenus.list'), 'r') \ as f: - vm.features['default-whitelist'] = ' '.join([x.rstrip() for x in f]) + vm.features['default-menu-items'] = ' '.join([x.rstrip() for x in f]) with open(os.path.join(source_dir, 'whitelisted-appmenus.list'), 'r') \ as f: - vm.features['whitelist'] = ' '.join([x.rstrip() for x in f]) + vm.features['menu-items'] = ' '.join([x.rstrip() for x in f]) with open( os.path.join(source_dir, 'netvm-whitelisted-appmenus.list'), 'r') \ as f: - vm.features['netvm-whitelist'] = ' '.join([x.rstrip() for x in f]) + vm.features['netvm-menu-items'] = ' '.join([x.rstrip() for x in f]) # TODO: change this to qrexec calls to GUI VM, when GUI VM will be # implemented From b634c7c7850c57a9a9ebf12f8fdc3e6598539396 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Thu, 9 Jul 2020 02:23:19 +0800 Subject: [PATCH 006/119] Initial commit of qvm-template. Refer to for previous revisions. --- qubesadmin/tools/qvm_template.py | 414 +++++++++++++++++++++++++++++++ 1 file changed, 414 insertions(+) create mode 100644 qubesadmin/tools/qvm_template.py diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py new file mode 100644 index 0000000..293690f --- /dev/null +++ b/qubesadmin/tools/qvm_template.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python3 + +import argparse +import datetime +import os +import shutil +import subprocess +import sys +import tempfile +import time + +import dnf +import qubesadmin +import rpm +import xdg.BaseDirectory + +PATH_PREFIX = '/var/lib/qubes/vm-templates' +TEMP_DIR = '/var/tmp' +PACKAGE_NAME_PREFIX = 'qubes-template-' +CACHE_DIR = os.path.join(xdg.BaseDirectory.xdg_cache_home, 'qvm-template') + +def qubes_release(): + try: + os_id = subprocess.check_output(['lsb_release', '-si'], + encoding='UTF-8').strip() + if os_id == 'Qubes': + return subprocess.check_output(['lsb_release', '-sr'], + encoding='UTF-8').strip() + except: + pass + with open('/usr/share/qubes/marker-vm', 'r') as f: + # Get last line (in the format `x.x`) + return f.readlines[-1].strip() + +parser = argparse.ArgumentParser(description='Qubes Template Manager') +parser.add_argument('operation', type=str) +parser.add_argument('templates', nargs='*') + +# qrexec related +parser.add_argument('--repo-files', action='append', + default=['/etc/yum.repos.d/qubes-templates.repo'], + help='Specify files containing DNF repository configuration.') +parser.add_argument('--updatevm', default='sys-firewall', + help='Specify VM to download updates from.') +# DNF-related options +parser.add_argument('--enablerepo', action='append', + help='Enable additional repositories.') +parser.add_argument('--disablerepo', action='append', + help='Disable certain repositories.') +parser.add_argument('--repoid', action='append', + help='Enable just specific repositories.') +parser.add_argument('--releasever', default=qubes_release(), + help='Override distro release version.') +parser.add_argument('--cachedir', default=CACHE_DIR, + help='Override cache directory.') +# qvm-template install +parser.add_argument('--nogpgcheck', action='store_true', + help='Disable signature checks.') +parser.add_argument('--allow-pv', action='store_true', + help='Allow setting virt_mode to pv in configuration file.') +# qvm-template download +parser.add_argument('--downloaddir', default='.', + help='Override download directory.') +# qvm-template list +parser.add_argument('--all', action='store_true') +parser.add_argument('--installed', action='store_true') +parser.add_argument('--available', action='store_true') +parser.add_argument('--extras', action='store_true') +parser.add_argument('--upgrades', action='store_true') + +# NOTE: Verifying RPMs this way is prone to TOCTOU. This is okay for local +# files, but may create problems if multiple instances of `qvm-template` are +# downloading the same file, so a lock is needed in that case. +def verify_rpm(path, nogpgcheck=False, transaction_set=None): + if transaction_set is None: + transaction_set = rpm.TransactionSet() + with open(path, 'rb') as f: + try: + hdr = transaction_set.hdrFromFdno(f) + if hdr[rpm.RPMTAG_SIGSIZE] is None \ + and hdr[rpm.RPMTAG_SIGPGP] is None \ + and hdr[rpm.RPMTAG_SIGGPG] is None: + return nogpgcheck + except rpm.error as e: + if str(e) == 'public key not trusted' \ + or str(e) == 'public key not available': + return nogpgcheck + return False + return True + +def get_package_hdr(path, transaction_set=None): + if transaction_set is None: + transaction_set = rpm.TransactionSet() + with open(path, 'rb') as f: + hdr = transaction_set.hdrFromFdno(f) + return hdr + +def extract_rpm(name, path, target): + with open(path, 'rb') as in_file: + rpm2cpio = subprocess.Popen(['rpm2cpio', path], stdout=subprocess.PIPE) + # `-D` is GNUism + cpio = subprocess.Popen([ + 'cpio', + '-idm', + '-D', + target, + '.%s/%s/*' % (PATH_PREFIX, name) + ], stdin=rpm2cpio.stdout, stdout=subprocess.DEVNULL) + return rpm2cpio.wait() == 0 and cpio.wait() == 0 + +def parse_config(path): + with open(path, 'r') as f: + return dict(line.rstrip('\n').split('=', 1) for line in f) + +def install(args, app): + # TODO: Lock, mentioned in the note above + transaction_set = rpm.TransactionSet() + + rpm_list = [] + for template in args.templates: + if template.endswith('.rpm'): + if not os.path.exists(template): + parser.error('RPM file \'%s\' not found.' % template) + rpm_list.append(template) + + os.makedirs(args.cachedir, exist_ok=True) + + dl_list = get_dl_list(args, app) + dl_list_copy = dl_list.copy() + # Verify that the templates are not yet installed + for name, ver in dl_list.items(): + if name in app.domains: + print(('Template \'%s\' already installed, skipping...' + ' (You may want to use the {reinstall,upgrade,downgrade}' + ' operations.)') % name, file=sys.stderr) + del dl_list_copy[name] + else: + version_str = build_version_str(ver) + target_file = \ + '%s%s-%s.rpm' % (PACKAGE_NAME_PREFIX, name, version_str) + rpm_list.append(os.path.join(args.cachedir, target_file)) + dl_list = dl_list_copy + + download(args, app, path_override=args.cachedir, dl_list=dl_list) + + for rpmfile in rpm_list: + if not verify_rpm(rpmfile, args.nogpgcheck, transaction_set): + parser.error('Package \'%s\' verification failed.' % template) + + for rpmfile in rpm_list: + with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: + package_hdr = get_package_hdr(rpmfile) + package_name = package_hdr[rpm.RPMTAG_NAME] + if not package_name.startswith(PACKAGE_NAME_PREFIX): + parser.error( + 'Illegal package name for package \'%s\'.' % rpmfile) + # Remove prefix to get the real template name + name = package_name[len(PACKAGE_NAME_PREFIX):] + + # Another check for already-downloaded RPMs + if name in app.domains: + print(('Template \'%s\' already installed, skipping...' + ' (You may want to use the {reinstall,upgrade,downgrade}' + ' operations.)') % name, file=sys.stderr) + continue + + print('Installing template \'%s\'...' % name, file=sys.stderr) + extract_rpm(name, rpmfile, target) + cmdline = [ + 'qvm-template-postprocess', + '--keep-source', + '--really', + '--no-installed-by-rpm', + ] + if args.allow_pv: + cmdline.append('--allow-pv') + subprocess.check_call(cmdline + [ + 'post-install', + name, + target + PATH_PREFIX + '/' + name]) + + app.domains.refresh_cache(force=True) + tpl = app.domains[name] + + tpl.features['template-epoch'] = \ + package_hdr[rpm.RPMTAG_EPOCHNUM] + tpl.features['template-version'] = \ + package_hdr[rpm.RPMTAG_VERSION] + tpl.features['template-release'] = \ + package_hdr[rpm.RPMTAG_RELEASE] + tpl.features['template-install-date'] = \ + str(datetime.datetime.today()) + tpl.features['template-name'] = name + # TODO: Store source repo + +def qrexec_popen(args, app, service, stdout=subprocess.PIPE, encoding='UTF-8'): + if args.updatevm: + from_vm = shutil.which('qrexec-client-vm') is not None + if from_vm: + return subprocess.Popen( + ['qrexec-client-vm', args.updatevm, service], + stdin=subprocess.PIPE, + stdout=stdout, + stderr=subprocess.PIPE, + encoding=encoding) + else: + return subprocess.Popen([ + 'qrexec-client', + '-d', + args.updatevm, + 'user:/etc/qubes-rpc/%s' % service + ], + stdin=subprocess.PIPE, + stdout=stdout, + stderr=subprocess.PIPE, + encoding=encoding) + else: + return subprocess.Popen([ + '/etc/qubes-rpc/%s' % service, + ], + stdin=subprocess.PIPE, + stdout=stdout, + stderr=subprocess.PIPE, + encoding=encoding) + +def qrexec_payload(args, app, spec): + payload = '' + for r in args.enablerepo if args.enablerepo else []: + payload += '--enablerepo=%s\n' % r + for r in args.disablerepo if args.disablerepo else []: + payload += '--disablerepo=%s\n' % r + for r in args.repoid if args.repoid else []: + payload += '--repoid=%s\n' % r + payload += '--releasever=%s\n' % args.releasever + payload += spec + '\n' + payload += '---\n' + for fn in args.repo_files: + with open(fn, 'r') as f: + payload += f.read() + '\n' + return payload + +def qrexec_repoquery(args, app, spec='*'): + proc = qrexec_popen(args, app, 'qubes.TemplateSearch') + payload = qrexec_payload(args, app, spec) + stdout, stderr = proc.communicate(payload) + if proc.wait() != 0: + return None + result = [] + for line in stdout.strip().split('\n'): + entry = line.split(':') + if not entry[0].startswith(PACKAGE_NAME_PREFIX): + continue + entry[0] = entry[0][len(PACKAGE_NAME_PREFIX):] + result.append(entry) + return result + +def qrexec_download(args, app, spec, path): + with open(path, 'wb') as f: + proc = qrexec_popen(args, app, 'qubes.TemplateDownload', + stdout=f, encoding=None) + payload = qrexec_payload(args, app, spec) + proc.stdin.write(payload.encode('UTF-8')) + proc.stdin.close() + while True: + c = proc.stderr.read(1) + if not c: + break + # Write raw byte w/o decoding + sys.stdout.buffer.write(c) + sys.stdout.flush() + if proc.wait() != 0: + return False + return True + +def build_version_str(evr): + return '%s:%s-%s' % evr + +def pretty_print_table(table): + if len(table) != 0: + widths = [] + for i in range(len(table[0])): + widths.append(max(len(s[i]) for s in table)) + for row in sorted(table): + cols = ['{key:{width}s}'.format( + key=row[i], width=widths[i]) for i in range(len(row))] + print(' '.join(cols)) + +def do_list(args, app): + # TODO: Check local template name actually matches to account for renames + # TODO: Also display repo like `dnf list` + tpl_list = [] + + if not (args.installed or args.available or args.extras or args.upgrades): + args.all = True + + if args.installed or args.all: + for vm in app.domains: + if 'template-install-date' in vm.features: + version_str = build_version_str(( + vm.features['template-epoch'], + vm.features['template-version'], + vm.features['template-release'])) + tpl_list.append((vm.name, version_str)) + + if args.available or args.all: + query_res = qrexec_repoquery(args, app) + for name, epoch, version, release, reponame, dlsize, summary \ + in query_res: + version_str = build_version_str((epoch, version, release)) + tpl_list.append((name, version_str)) + + if args.extras: + query_res = qrexec_repoquery(args, app) + remote = set() + for name, epoch, version, release, reponame, dlsize, summary \ + in query_res: + remote.add(name) + for vm in app.domains: + if 'template-name' in vm.features and \ + vm.features['template-name'] not in remote: + version_str = build_version_str(( + vm.features['template-epoch'], + vm.features['template-version'], + vm.features['template-release'])) + tpl_list.append((vm.name, version_str)) + + if args.upgrades: + query_res = qrexec_repoquery(args, app) + local = {} + for vm in app.domains: + if 'template-name' in vm.features: + local[vm.features['template-name']] = ( + vm.features['template-epoch'], + vm.features['template-version'], + vm.features['template-release']) + for name, epoch, version, release, reponame, dlsize, summary \ + in query_res: + if name in local: + if rpm.labelCompare(local[name], (epoch, version, release)) < 0: + version_str = build_version_str((epoch, version, release)) + tpl_list.append((name, version_str)) + + pretty_print_table(tpl_list) + +def get_dl_list(args, app): + candid = {} + for template in args.templates: + # Skip local RPMs + if template.endswith('.rpm'): + continue + query_res = qrexec_repoquery(args, app, PACKAGE_NAME_PREFIX + template) + if len(query_res) == 0: + parser.error('Package \'%s\' not found.' % template) + # TODO: Better error handling + sys.exit(1) + # We only select one (latest) package for each distinct package name + for name, epoch, version, release, reponame, dlsize, summary \ + in query_res: + ver = (epoch, version, release) + if name not in candid or rpm.labelCompare(candid[name], ver) < 0: + candid[name] = ver + return candid + +def download(args, app, path_override=None, dl_list=None): + if dl_list is None: + dl_list = get_dl_list(args, app) + + path = path_override if path_override != None else args.downloaddir + for name, ver in dl_list.items(): + version_str = build_version_str(ver) + spec = PACKAGE_NAME_PREFIX + name + '-' + version_str + target = os.path.join(path, '%s.rpm' % spec) + if os.path.exists(target): + print('\'%s\' already exists, skipping...' % target, + file=sys.stderr) + else: + print('Downloading \'%s\'...' % spec, file=sys.stderr) + ret = qrexec_download(args, app, spec, target) + if not ret: + # TODO: Retry? + print('\'%s\' download failed.' % spec, file=sys.stderr) + sys.exit(1) + +def remove(args, app): + # Use exec so stdio can be shared easily + os.execvp('qvm-remove', ['qvm-remove'] + args.templates) + +def clean(args, app): + # TODO: More fine-grained options + shutil.rmtree(args.cachedir) + +def main(args=None, app=None): + args = parser.parse_args(args) + + if app is None: + app = qubesadmin.Qubes() + + if args.operation == 'install': + install(args, app) + elif args.operation == 'list': + do_list(args, app) + elif args.operation == 'download': + download(args, app) + elif args.operation == 'remove': + remove(args, app) + elif args.operation == 'clean': + clean(args, app) + else: + parser.error('Operation \'%s\' not supported.' % args.operation) + + return 0 + +if __name__ == '__main__': + sys.exit(main()) From 0e8e8d98de53285d84c6327174ee3527048e4d8e Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Thu, 9 Jul 2020 02:38:36 +0800 Subject: [PATCH 007/119] Better way of detecting VM. --- qubesadmin/tools/qvm_template.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 293690f..3d3ad0a 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -20,17 +20,12 @@ PACKAGE_NAME_PREFIX = 'qubes-template-' CACHE_DIR = os.path.join(xdg.BaseDirectory.xdg_cache_home, 'qvm-template') def qubes_release(): - try: - os_id = subprocess.check_output(['lsb_release', '-si'], - encoding='UTF-8').strip() - if os_id == 'Qubes': - return subprocess.check_output(['lsb_release', '-sr'], - encoding='UTF-8').strip() - except: - pass - with open('/usr/share/qubes/marker-vm', 'r') as f: - # Get last line (in the format `x.x`) - return f.readlines[-1].strip() + if os.path.exists('/usr/share/qubes/marker-vm'): + with open('/usr/share/qubes/marker-vm', 'r') as f: + # Get last line (in the format `x.x`) + return f.readlines()[-1].strip() + return subprocess.check_output(['lsb_release', '-sr'], + encoding='UTF-8').strip() parser = argparse.ArgumentParser(description='Qubes Template Manager') parser.add_argument('operation', type=str) From 3d42c988f04e913eb0aee0c54ea0dc303a691383 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Fri, 10 Jul 2020 02:43:03 +0800 Subject: [PATCH 008/119] Various cleanup and improvements. - `qvm-template list`: show template state - `qvm-template list`: only call qubes.TemplateSearch once - `qvm-template list`: use `qubesadmin.tools.print_table()` instead of own implementation - `qvm-template download`: custom progress bar - Use `run_service` instead of own implementation - Remove some erroneous/redundant lines --- qubesadmin/tools/qvm_template.py | 130 +++++++++++++++---------------- 1 file changed, 64 insertions(+), 66 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 3d3ad0a..bcb4d1b 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -2,6 +2,8 @@ import argparse import datetime +import enum +import math import os import shutil import subprocess @@ -11,6 +13,7 @@ import time import dnf import qubesadmin +import qubesadmin.tools import rpm import xdg.BaseDirectory @@ -63,6 +66,12 @@ parser.add_argument('--available', action='store_true') parser.add_argument('--extras', action='store_true') parser.add_argument('--upgrades', action='store_true') +class TemplateState(enum.Enum): + INSTALLED = 'installed' + AVAILABLE = 'available' + EXTRA = 'extra' + UPGRADABLE = 'upgradable' + # NOTE: Verifying RPMs this way is prone to TOCTOU. This is okay for local # files, but may create problems if multiple instances of `qvm-template` are # downloading the same file, so a lock is needed in that case. @@ -91,17 +100,16 @@ def get_package_hdr(path, transaction_set=None): return hdr def extract_rpm(name, path, target): - with open(path, 'rb') as in_file: - rpm2cpio = subprocess.Popen(['rpm2cpio', path], stdout=subprocess.PIPE) - # `-D` is GNUism - cpio = subprocess.Popen([ - 'cpio', - '-idm', - '-D', - target, - '.%s/%s/*' % (PATH_PREFIX, name) - ], stdin=rpm2cpio.stdout, stdout=subprocess.DEVNULL) - return rpm2cpio.wait() == 0 and cpio.wait() == 0 + rpm2cpio = subprocess.Popen(['rpm2cpio', path], stdout=subprocess.PIPE) + # `-D` is GNUism + cpio = subprocess.Popen([ + 'cpio', + '-idm', + '-D', + target, + '.%s/%s/*' % (PATH_PREFIX, name) + ], stdin=rpm2cpio.stdout, stdout=subprocess.DEVNULL) + return rpm2cpio.wait() == 0 and cpio.wait() == 0 def parse_config(path): with open(path, 'r') as f: @@ -123,7 +131,7 @@ def install(args, app): dl_list = get_dl_list(args, app) dl_list_copy = dl_list.copy() # Verify that the templates are not yet installed - for name, ver in dl_list.items(): + for name, (ver, _) in dl_list.items(): if name in app.domains: print(('Template \'%s\' already installed, skipping...' ' (You may want to use the {reinstall,upgrade,downgrade}' @@ -163,7 +171,6 @@ def install(args, app): extract_rpm(name, rpmfile, target) cmdline = [ 'qvm-template-postprocess', - '--keep-source', '--really', '--no-installed-by-rpm', ] @@ -188,35 +195,19 @@ def install(args, app): tpl.features['template-name'] = name # TODO: Store source repo -def qrexec_popen(args, app, service, stdout=subprocess.PIPE, encoding='UTF-8'): +def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): if args.updatevm: - from_vm = shutil.which('qrexec-client-vm') is not None - if from_vm: - return subprocess.Popen( - ['qrexec-client-vm', args.updatevm, service], - stdin=subprocess.PIPE, - stdout=stdout, - stderr=subprocess.PIPE, - encoding=encoding) - else: - return subprocess.Popen([ - 'qrexec-client', - '-d', - args.updatevm, - 'user:/etc/qubes-rpc/%s' % service - ], - stdin=subprocess.PIPE, - stdout=stdout, - stderr=subprocess.PIPE, - encoding=encoding) + return app.domains[args.updatevm].run_service( + service, + filter_esc=filter_esc, + stdout=stdout) else: return subprocess.Popen([ '/etc/qubes-rpc/%s' % service, ], stdin=subprocess.PIPE, stdout=stdout, - stderr=subprocess.PIPE, - encoding=encoding) + stderr=subprocess.PIPE) def qrexec_payload(args, app, spec): payload = '' @@ -237,7 +228,8 @@ def qrexec_payload(args, app, spec): def qrexec_repoquery(args, app, spec='*'): proc = qrexec_popen(args, app, 'qubes.TemplateSearch') payload = qrexec_payload(args, app, spec) - stdout, stderr = proc.communicate(payload) + stdout, stderr = proc.communicate(payload.encode('UTF-8')) + stdout = stdout.decode('ASCII') if proc.wait() != 0: return None result = [] @@ -249,20 +241,33 @@ def qrexec_repoquery(args, app, spec='*'): result.append(entry) return result -def qrexec_download(args, app, spec, path): +def qrexec_download(args, app, spec, path, dlsize=None): with open(path, 'wb') as f: + # Don't filter ESCs for binary files proc = qrexec_popen(args, app, 'qubes.TemplateDownload', - stdout=f, encoding=None) + stdout=f, filter_esc=False) payload = qrexec_payload(args, app, spec) proc.stdin.write(payload.encode('UTF-8')) proc.stdin.close() - while True: - c = proc.stderr.read(1) - if not c: - break - # Write raw byte w/o decoding - sys.stdout.buffer.write(c) - sys.stdout.flush() + while proc.poll() == None: + width = shutil.get_terminal_size((80, 20)).columns + pct = '%5.f%% ' % math.floor(f.tell() / dlsize * 100) + bar_len = width - len(pct) - 2 + num_hash = math.floor(f.tell() / dlsize * bar_len) + num_space = bar_len - num_hash + # Clear previous bar + print(u'\u001b[1000D', end='', file=sys.stderr) + print(pct + '[' + ('#' * num_hash) + (' ' * num_space) + ']', + end='', file=sys.stderr) + sys.stderr.flush() + time.sleep(0.1) + #while True: + # c = proc.stderr.read(1) + # if not c: + # break + # # Write raw byte w/o decoding + # sys.stdout.buffer.write(c) + # sys.stdout.flush() if proc.wait() != 0: return False return True @@ -270,16 +275,6 @@ def qrexec_download(args, app, spec, path): def build_version_str(evr): return '%s:%s-%s' % evr -def pretty_print_table(table): - if len(table) != 0: - widths = [] - for i in range(len(table[0])): - widths.append(max(len(s[i]) for s in table)) - for row in sorted(table): - cols = ['{key:{width}s}'.format( - key=row[i], width=widths[i]) for i in range(len(row))] - print(' '.join(cols)) - def do_list(args, app): # TODO: Check local template name actually matches to account for renames # TODO: Also display repo like `dnf list` @@ -288,6 +283,9 @@ def do_list(args, app): if not (args.installed or args.available or args.extras or args.upgrades): args.all = True + if args.all or args.available or args.extras or args.upgrades: + query_res = qrexec_repoquery(args, app) + if args.installed or args.all: for vm in app.domains: if 'template-install-date' in vm.features: @@ -295,17 +293,16 @@ def do_list(args, app): vm.features['template-epoch'], vm.features['template-version'], vm.features['template-release'])) - tpl_list.append((vm.name, version_str)) + tpl_list.append( + (vm.name, version_str, TemplateState.INSTALLED.value)) if args.available or args.all: - query_res = qrexec_repoquery(args, app) for name, epoch, version, release, reponame, dlsize, summary \ in query_res: version_str = build_version_str((epoch, version, release)) - tpl_list.append((name, version_str)) + tpl_list.append((name, version_str, TemplateState.AVAILABLE.value)) if args.extras: - query_res = qrexec_repoquery(args, app) remote = set() for name, epoch, version, release, reponame, dlsize, summary \ in query_res: @@ -317,10 +314,10 @@ def do_list(args, app): vm.features['template-epoch'], vm.features['template-version'], vm.features['template-release'])) - tpl_list.append((vm.name, version_str)) + tpl_list.append( + (vm.name, version_str, TemplateState.EXTRA.value)) if args.upgrades: - query_res = qrexec_repoquery(args, app) local = {} for vm in app.domains: if 'template-name' in vm.features: @@ -333,9 +330,10 @@ def do_list(args, app): if name in local: if rpm.labelCompare(local[name], (epoch, version, release)) < 0: version_str = build_version_str((epoch, version, release)) - tpl_list.append((name, version_str)) + tpl_list.append( + (name, version_str, TemplateState.UPGRADABLE.value)) - pretty_print_table(tpl_list) + qubesadmin.tools.print_table(tpl_list) def get_dl_list(args, app): candid = {} @@ -353,7 +351,7 @@ def get_dl_list(args, app): in query_res: ver = (epoch, version, release) if name not in candid or rpm.labelCompare(candid[name], ver) < 0: - candid[name] = ver + candid[name] = (ver, int(dlsize)) return candid def download(args, app, path_override=None, dl_list=None): @@ -361,7 +359,7 @@ def download(args, app, path_override=None, dl_list=None): dl_list = get_dl_list(args, app) path = path_override if path_override != None else args.downloaddir - for name, ver in dl_list.items(): + for name, (ver, dlsize) in dl_list.items(): version_str = build_version_str(ver) spec = PACKAGE_NAME_PREFIX + name + '-' + version_str target = os.path.join(path, '%s.rpm' % spec) @@ -370,7 +368,7 @@ def download(args, app, path_override=None, dl_list=None): file=sys.stderr) else: print('Downloading \'%s\'...' % spec, file=sys.stderr) - ret = qrexec_download(args, app, spec, target) + ret = qrexec_download(args, app, spec, target, dlsize) if not ret: # TODO: Retry? print('\'%s\' download failed.' % spec, file=sys.stderr) From 73eb4cd08cad3ce6c7cb6ba46c705ecee6d59930 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 11 Jul 2020 21:42:58 +0800 Subject: [PATCH 009/119] Use tqdm for progress bar. --- qubesadmin/tools/.qvm_template.py.swp | Bin 0 -> 16384 bytes qubesadmin/tools/qvm_template.py | 28 +++++++++----------------- 2 files changed, 9 insertions(+), 19 deletions(-) create mode 100644 qubesadmin/tools/.qvm_template.py.swp diff --git a/qubesadmin/tools/.qvm_template.py.swp b/qubesadmin/tools/.qvm_template.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..9d3f470910e469595b12829c9ab4303f12b4e74b GIT binary patch literal 16384 zcmeHOTZklA8LlL|u5NZ!6a)jxS%+BNak@Ge(PW{K&P>l_aOaYlo;754tgWs(-Ca9% z%Q>ferV$h(@m&#&K8PYJf~+LBR`%82$d!Roykc)00^dALJDK zO?RL3U(Wf?f9|!N2d}TKvqznWEj(|wtbYU-+;6<&ZtKUNvaEsRf|6hB=?uass68|2 zaCxfeN6BDa%hj~b}px4va}Cg1!t3>XFs1BL;^fMLKeU>GnA7zX~I8Bp1I z>tm?k-FYRu`R83Tpa06A{XD%llmBX-ALQxZo5}xYp8s^7UdbI4e(uTB-@3)^~u<0h_=E@EC9o_~B*C z`V4Rjr~{XQOThOpS=MKP7>Iy9;N8F<@3pM20|ICQ?*aaH(XxI7d>Xg~JO*3@UVMXP zeE~QCT7U!m_4StZQ{Z{vL%;`s%fN3hSk}*g&jVe+0sj6vlpFXq@I~M`AOv=S_XF<( z{%{^TfD2p!?gCyqXIb9_z6>Z}3AhNnghP%OfKLMZ00Umd!OQP}UjRP`z65+6XanyB zE&zW)+x!`zHhPiTY3>v7F5^c^c)oH)7FQB4=8G)zRDY2vPxQE|UWMQM5nK&K!uo-j zB;h-2Fc?TAY=mjzO0FEFu81^V36iwZt!m-JQ1w}w@uV`gZi|jxWu9c+s}o~0W?&d) za@telJ6u)9=ECl*Qx;0LktY1AMU_Iy-OTe3ydHN^n%wD(g9gbL70fHw76U#h5W|zF zuXvF>o>9t7(q7i{``kY`rb-g@q=HE$D!j@X4Q6M9P89m=fR9*`D&`HnF!DMPx2Hsq z5^6YKNr{0}Jh0gfi)88HM?TM#W{e8~m*5s;=)BUOygIK@VlN4(nfHkQ?XC58bJg8h zTV35+-JjB2GCKKsvfkF})nu~%wtjNA5bGCoCiE$ zVFD>nG2fHiu@1gvFBhjt7wnvxq}gDlhT5WNDY(#Jf_p z;$5km%o7~OHa6R>tL(b24V4F>AmI+)4`CZ0u$yqEQr6yFYH#j3tWBBtmmWZ%izq~K zy=Xjy>~5`YHoz}eGT)S2%OgoCsTie)a zH?Li5Et8I}N)(|-SbpS5$=dl_Uporbukd6LJ1zX7&YpP!ua!Ao5V#oO1~J|otJt;L zxIA3ii)=iSHEgNyikXYP#_j4nHC@^}Rhm}i&YVrFd1rcF6wRyF0xLH>i$bY1y?q{K z4SN%jEW&_KqLKxn;68X6IrBLwLKw1ei{aQq$Te)oUPMeCb5}@~TsEU9J!I$-iXv_p z3gyNrN`N5Ch7|68nsme7KzKCW=y*S+D5!aQSj&bdSAVhOcR7?4j7)~T#KSztnI6#P zDaNP0dY<{2>gMfN@;V{d_u%eK>}tv5)Gjrbo@lwtYun`tO-u)9Q;}jKi#P@eh9;F@ z?v#4vk`OF4pMDL~&9f@ivZq#}nY)EKiOG!nVHfk8;8`j|L`uj{mmecSNHOI_XZ65!#pgL#V9OgTH86%DD8Sl>Y|B!$ zb~IHZc72_KyXXpRkOfe{!U!vU`c8yAj)u3gP^kre#9ws zG-Y#!>WMelNHBuShW)_2u?P#!@i+llHOhDcJ)z3-;$h-|x}FZ4Wa>B*rIaOsOs zw^+k|l3rAv%qVQ#OR%aGaH-iLhIrzqG+{UF-S$fDQM>BEPnhLv)qKe%2OUh8m~N$` zM;137sBEArdv+u{MaM!p?V1$CAsH^7+(Z`ydqET?Tvn=kwTJeP`6+}K*eX{H(;)sp zlvVT^0gH%K#8@fZJaUe1j$4*Arb{EcK9Hg=`rall1Q2bql;m)7f8NTkxC z)s4fCFu=&lY3i7FX;nTmhF~(Dj9h4p!rrYkOe#k~&v_if)$(M%L5i!#KDVFJF2yd- zBxsGT=F$h7t1WkGnA7zPXjh5^HXVZbo(-(-MJ1F*TI{k*EgI3TK2s}}b8 zu1nW~uB&gaT$gfO7q?XwDdZ;;_&=ib3r~K$uJ52K_GWPa!*1C3%Nw@7@{)s$_Rl Date: Sat, 11 Jul 2020 22:08:16 +0800 Subject: [PATCH 010/119] Check for newlines in qrexec arguments & improve error handling. --- qubesadmin/tools/qvm_template.py | 33 ++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index abdf937..5cd6c2c 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -60,6 +60,8 @@ parser.add_argument('--allow-pv', action='store_true', # qvm-template download parser.add_argument('--downloaddir', default='.', help='Override download directory.') +parser.add_argument('--retries', default=5, type=int, + help='Override number of retries for downloads.') # qvm-template list parser.add_argument('--all', action='store_true') parser.add_argument('--installed', action='store_true') @@ -211,14 +213,24 @@ def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): stderr=subprocess.PIPE) def qrexec_payload(args, app, spec): + def check_newline(string, name): + if '\n' in string: + parser.error(f"Malformed {name}:" + + " argument should not contain '\\n'.") + payload = '' for r in args.enablerepo if args.enablerepo else []: + check_newline(r, '--enablerepo') payload += '--enablerepo=%s\n' % r for r in args.disablerepo if args.disablerepo else []: + check_newline(r, '--disablerepo') payload += '--disablerepo=%s\n' % r for r in args.repoid if args.repoid else []: + check_newline(r, '--repoid') payload += '--repoid=%s\n' % r + check_newline(args.releasever, '--releasever') payload += '--releasever=%s\n' % args.releasever + check_newline(spec, 'template name') payload += spec + '\n' payload += '---\n' for fn in args.repo_files: @@ -232,7 +244,7 @@ def qrexec_repoquery(args, app, spec='*'): stdout, stderr = proc.communicate(payload.encode('UTF-8')) stdout = stdout.decode('ASCII') if proc.wait() != 0: - return None + raise ConnectionError("qrexec call 'qubes.TemplateSearch' failed.") result = [] for line in stdout.strip().split('\n'): entry = line.split(':') @@ -259,7 +271,8 @@ def qrexec_download(args, app, spec, path, dlsize=None): last = cur time.sleep(0.1) if proc.wait() != 0: - return False + raise ConnectionError( + "qrexec call 'qubes.TemplateDownload' failed.") return True def build_version_str(evr): @@ -334,7 +347,6 @@ def get_dl_list(args, app): query_res = qrexec_repoquery(args, app, PACKAGE_NAME_PREFIX + template) if len(query_res) == 0: parser.error('Package \'%s\' not found.' % template) - # TODO: Better error handling sys.exit(1) # We only select one (latest) package for each distinct package name for name, epoch, version, release, reponame, dlsize, summary \ @@ -358,10 +370,19 @@ def download(args, app, path_override=None, dl_list=None): file=sys.stderr) else: print('Downloading \'%s\'...' % spec, file=sys.stderr) - ret = qrexec_download(args, app, spec, target, dlsize) - if not ret: - # TODO: Retry? + ok = False + for attempt in range(args.retries): + try: + qrexec_download(args, app, spec, target, dlsize) + ok = True + break + except ConnectionError: + if attempt + 1 < args.retries: + print('\'%s\' download failed, retrying...' % spec, + file=sys.stderr) + if not ok: print('\'%s\' download failed.' % spec, file=sys.stderr) + os.remove(target) sys.exit(1) def remove(args, app): From 8a4b5e683ad42019bbbcf519225c3095592b3063 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 11 Jul 2020 22:31:38 +0800 Subject: [PATCH 011/119] Add suffix for unverified RPMs. --- qubesadmin/tools/qvm_template.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 5cd6c2c..2661239 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -22,6 +22,7 @@ PATH_PREFIX = '/var/lib/qubes/vm-templates' TEMP_DIR = '/var/tmp' PACKAGE_NAME_PREFIX = 'qubes-template-' CACHE_DIR = os.path.join(xdg.BaseDirectory.xdg_cache_home, 'qvm-template') +UNVERIFIED_SUFFIX = '.unverified' def qubes_release(): if os.path.exists('/usr/share/qubes/marker-vm'): @@ -147,11 +148,14 @@ def install(args, app): rpm_list.append(os.path.join(args.cachedir, target_file)) dl_list = dl_list_copy - download(args, app, path_override=args.cachedir, dl_list=dl_list) + download(args, app, path_override=args.cachedir, + dl_list=dl_list, suffix=UNVERIFIED_SUFFIX) - for rpmfile in rpm_list: - if not verify_rpm(rpmfile, args.nogpgcheck, transaction_set): + for idx, rpmfile in enumerate(rpm_list): + path = rpmfile + UNVERIFIED_SUFFIX + if not verify_rpm(path, args.nogpgcheck, transaction_set): parser.error('Package \'%s\' verification failed.' % template) + os.rename(path, rpmfile) for rpmfile in rpm_list: with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: @@ -356,7 +360,7 @@ def get_dl_list(args, app): candid[name] = (ver, int(dlsize)) return candid -def download(args, app, path_override=None, dl_list=None): +def download(args, app, path_override=None, dl_list=None, suffix=''): if dl_list is None: dl_list = get_dl_list(args, app) @@ -365,15 +369,21 @@ def download(args, app, path_override=None, dl_list=None): version_str = build_version_str(ver) spec = PACKAGE_NAME_PREFIX + name + '-' + version_str target = os.path.join(path, '%s.rpm' % spec) + target_suffix = target + suffix + if suffix != '' and os.path.exists(target_suffix): + print('\'%s\' already exists, skipping...' % target, + file=sys.stderr) if os.path.exists(target): print('\'%s\' already exists, skipping...' % target, file=sys.stderr) + if suffix != '': + os.rename(target, target_suffix) else: print('Downloading \'%s\'...' % spec, file=sys.stderr) ok = False for attempt in range(args.retries): try: - qrexec_download(args, app, spec, target, dlsize) + qrexec_download(args, app, spec, target_suffix, dlsize) ok = True break except ConnectionError: @@ -382,7 +392,7 @@ def download(args, app, path_override=None, dl_list=None): file=sys.stderr) if not ok: print('\'%s\' download failed.' % spec, file=sys.stderr) - os.remove(target) + os.remove(target_suffix) sys.exit(1) def remove(args, app): From faef52e61ad5cabd475b59e52b74040761f42c86 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 11 Jul 2020 23:12:57 +0800 Subject: [PATCH 012/119] Fix pylint warnings. --- qubesadmin/tools/.qvm_template.py.swp | Bin 16384 -> 0 bytes qubesadmin/tools/qvm_template.py | 95 ++++++++++++++------------ 2 files changed, 50 insertions(+), 45 deletions(-) delete mode 100644 qubesadmin/tools/.qvm_template.py.swp diff --git a/qubesadmin/tools/.qvm_template.py.swp b/qubesadmin/tools/.qvm_template.py.swp deleted file mode 100644 index 9d3f470910e469595b12829c9ab4303f12b4e74b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHOTZklA8LlL|u5NZ!6a)jxS%+BNak@Ge(PW{K&P>l_aOaYlo;754tgWs(-Ca9% z%Q>ferV$h(@m&#&K8PYJf~+LBR`%82$d!Roykc)00^dALJDK zO?RL3U(Wf?f9|!N2d}TKvqznWEj(|wtbYU-+;6<&ZtKUNvaEsRf|6hB=?uass68|2 zaCxfeN6BDa%hj~b}px4va}Cg1!t3>XFs1BL;^fMLKeU>GnA7zX~I8Bp1I z>tm?k-FYRu`R83Tpa06A{XD%llmBX-ALQxZo5}xYp8s^7UdbI4e(uTB-@3)^~u<0h_=E@EC9o_~B*C z`V4Rjr~{XQOThOpS=MKP7>Iy9;N8F<@3pM20|ICQ?*aaH(XxI7d>Xg~JO*3@UVMXP zeE~QCT7U!m_4StZQ{Z{vL%;`s%fN3hSk}*g&jVe+0sj6vlpFXq@I~M`AOv=S_XF<( z{%{^TfD2p!?gCyqXIb9_z6>Z}3AhNnghP%OfKLMZ00Umd!OQP}UjRP`z65+6XanyB zE&zW)+x!`zHhPiTY3>v7F5^c^c)oH)7FQB4=8G)zRDY2vPxQE|UWMQM5nK&K!uo-j zB;h-2Fc?TAY=mjzO0FEFu81^V36iwZt!m-JQ1w}w@uV`gZi|jxWu9c+s}o~0W?&d) za@telJ6u)9=ECl*Qx;0LktY1AMU_Iy-OTe3ydHN^n%wD(g9gbL70fHw76U#h5W|zF zuXvF>o>9t7(q7i{``kY`rb-g@q=HE$D!j@X4Q6M9P89m=fR9*`D&`HnF!DMPx2Hsq z5^6YKNr{0}Jh0gfi)88HM?TM#W{e8~m*5s;=)BUOygIK@VlN4(nfHkQ?XC58bJg8h zTV35+-JjB2GCKKsvfkF})nu~%wtjNA5bGCoCiE$ zVFD>nG2fHiu@1gvFBhjt7wnvxq}gDlhT5WNDY(#Jf_p z;$5km%o7~OHa6R>tL(b24V4F>AmI+)4`CZ0u$yqEQr6yFYH#j3tWBBtmmWZ%izq~K zy=Xjy>~5`YHoz}eGT)S2%OgoCsTie)a zH?Li5Et8I}N)(|-SbpS5$=dl_Uporbukd6LJ1zX7&YpP!ua!Ao5V#oO1~J|otJt;L zxIA3ii)=iSHEgNyikXYP#_j4nHC@^}Rhm}i&YVrFd1rcF6wRyF0xLH>i$bY1y?q{K z4SN%jEW&_KqLKxn;68X6IrBLwLKw1ei{aQq$Te)oUPMeCb5}@~TsEU9J!I$-iXv_p z3gyNrN`N5Ch7|68nsme7KzKCW=y*S+D5!aQSj&bdSAVhOcR7?4j7)~T#KSztnI6#P zDaNP0dY<{2>gMfN@;V{d_u%eK>}tv5)Gjrbo@lwtYun`tO-u)9Q;}jKi#P@eh9;F@ z?v#4vk`OF4pMDL~&9f@ivZq#}nY)EKiOG!nVHfk8;8`j|L`uj{mmecSNHOI_XZ65!#pgL#V9OgTH86%DD8Sl>Y|B!$ zb~IHZc72_KyXXpRkOfe{!U!vU`c8yAj)u3gP^kre#9ws zG-Y#!>WMelNHBuShW)_2u?P#!@i+llHOhDcJ)z3-;$h-|x}FZ4Wa>B*rIaOsOs zw^+k|l3rAv%qVQ#OR%aGaH-iLhIrzqG+{UF-S$fDQM>BEPnhLv)qKe%2OUh8m~N$` zM;137sBEArdv+u{MaM!p?V1$CAsH^7+(Z`ydqET?Tvn=kwTJeP`6+}K*eX{H(;)sp zlvVT^0gH%K#8@fZJaUe1j$4*Arb{EcK9Hg=`rall1Q2bql;m)7f8NTkxC z)s4fCFu=&lY3i7FX;nTmhF~(Dj9h4p!rrYkOe#k~&v_if)$(M%L5i!#KDVFJF2yd- zBxsGT=F$h7t1WkGnA7zPXjh5^HXVZbo(-(-MJ1F*TI{k*EgI3TK2s}}b8 zu1nW~uB&gaT$gfO7q?XwDdZ;;_&=ib3r~K$uJ52K_GWPa!*1C3%Nw@7@{)s$_Rl Date: Sun, 12 Jul 2020 23:02:07 +0800 Subject: [PATCH 013/119] Support for {reinstall,downgrade,upgrade} operations. Requires QubesOS/qubes-issues#5946 to be resolved. --- qubesadmin/tools/qvm_template.py | 100 ++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 15 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index ab73213..9a68600 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -74,6 +74,12 @@ class TemplateState(enum.Enum): EXTRA = 'extra' UPGRADABLE = 'upgradable' +class VersionSelector(enum.Enum): + LATEST = enum.auto() + REINSTALL = enum.auto() + LATEST_LOWER = enum.auto() + LATEST_HIGHER = enum.auto() + # NOTE: Verifying RPMs this way is prone to TOCTOU. This is okay for local # files, but may create problems if multiple instances of `qvm-template` are # downloading the same file, so a lock is needed in that case. @@ -117,7 +123,8 @@ def parse_config(path): with open(path, 'r') as fd: return dict(line.rstrip('\n').split('=', 1) for line in fd) -def install(args, app): +def install(args, app, version_selector=VersionSelector.LATEST, + ignore_existing=False): # TODO: Lock, mentioned in the note above transaction_set = rpm.TransactionSet() @@ -130,11 +137,11 @@ def install(args, app): os.makedirs(args.cachedir, exist_ok=True) - dl_list = get_dl_list(args, app) + dl_list = get_dl_list(args, app, version_selector=version_selector) dl_list_copy = dl_list.copy() # Verify that the templates are not yet installed for name, (ver, _) in dl_list.items(): - if name in app.domains: + if not ignore_existing and name in app.domains: print(('Template \'%s\' already installed, skipping...' ' (You may want to use the {reinstall,upgrade,downgrade}' ' operations.)') % name, file=sys.stderr) @@ -147,7 +154,8 @@ def install(args, app): dl_list = dl_list_copy download(args, app, path_override=args.cachedir, - dl_list=dl_list, suffix=UNVERIFIED_SUFFIX) + dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, + version_selector=version_selector) for rpmfile in rpm_list: path = rpmfile + UNVERIFIED_SUFFIX @@ -166,7 +174,7 @@ def install(args, app): name = package_name[len(PACKAGE_NAME_PREFIX):] # Another check for already-downloaded RPMs - if name in app.domains: + if not ignore_existing and name in app.domains: print(('Template \'%s\' already installed, skipping...' ' (You may want to use the {reinstall,upgrade,downgrade}' ' operations.)') % name, file=sys.stderr) @@ -342,28 +350,81 @@ def do_list(args, app): qubesadmin.tools.print_table(tpl_list) -def get_dl_list(args, app): - candid = {} +def get_dl_list(args, app, version_selector=VersionSelector.LATEST): + full_candid = {} for template in args.templates: + # This will be merged into `full_candid` later. + # It is separated so that we can check whether it is empty. + candid = {} + # Skip local RPMs if template.endswith('.rpm'): continue + query_res = qrexec_repoquery(args, app, PACKAGE_NAME_PREFIX + template) - if len(query_res) == 0: - parser.error('Package \'%s\' not found.' % template) - sys.exit(1) - # We only select one (latest) package for each distinct package name + + # We only select one package for each distinct package name #pylint: disable=unused-variable for name, epoch, version, release, reponame, dlsize, summary \ in query_res: ver = (epoch, version, release) - if name not in candid or rpm.labelCompare(candid[name], ver) < 0: - candid[name] = (ver, int(dlsize)) + if version_selector == VersionSelector.LATEST: + if name not in candid \ + or rpm.labelCompare(candid[name], ver) < 0: + candid[name] = (ver, int(dlsize)) + elif version_selector == VersionSelector.REINSTALL: + if name not in app.domains: + parser.error("Template '%s' not installed." % name) + vm = app.domains[name] + cur_ver = ( + vm.features['template-epoch'], + vm.features['template-version'], + vm.features['template-release']) + if rpm.labelCompare(ver, cur_ver) == 0: + candid[name] = (ver, int(dlsize)) + elif version_selector in [VersionSelector.LATEST_LOWER, + VersionSelector.LATEST_HIGHER]: + if name not in app.domains: + parser.error("Template '%s' not installed." % name) + vm = app.domains[name] + cur_ver = ( + vm.features['template-epoch'], + vm.features['template-version'], + vm.features['template-release']) + cmp_res = -1 \ + if version_selector == VersionSelector.LATEST_LOWER \ + else 1 + if rpm.labelCompare(ver, cur_ver) == cmp_res: + if name not in candid \ + or rpm.labelCompare(candid[name], ver) < 0: + candid[name] = (ver, int(dlsize)) + + if len(candid) == 0: + if version_selector == VersionSelector.LATEST: + parser.error('Template \'%s\' not found.' % template) + elif version_selector == VersionSelector.REINSTALL: + parser.error('Same version of template \'%s\' not found.' \ + % template) + elif version_selector == VersionSelector.LATEST_LOWER: + parser.error('Lower version of template \'%s\' not found.' \ + % template) + elif version_selector == VersionSelector.LATEST_HIGHER: + parser.error('Higher version of template \'%s\' not found.' \ + % template) + sys.exit(1) + + # Merge & choose the template with the highest version + for name, (ver, dlsize) in candid.items(): + if name not in full_candid \ + or rpm.labelCompare(full_candid[name], ver) < 0: + full_candid[name] = (ver, dlsize) + return candid -def download(args, app, path_override=None, dl_list=None, suffix=''): +def download(args, app, path_override=None, + dl_list=None, suffix='', version_selector=VersionSelector.LATEST): if dl_list is None: - dl_list = get_dl_list(args, app) + dl_list = get_dl_list(args, app, version_selector=version_selector) path = path_override if path_override is not None else args.downloaddir for name, (ver, dlsize) in dl_list.items(): @@ -416,6 +477,15 @@ def main(args=None, app=None): if args.operation == 'install': install(args, app) + elif args.operation == 'reinstall': + install(args, app, version_selector=VersionSelector.REINSTALL, + ignore_existing=True) + elif args.operation == 'downgrade': + install(args, app, version_selector=VersionSelector.LATEST_LOWER, + ignore_existing=True) + elif args.operation == 'upgrade': + install(args, app, version_selector=VersionSelector.LATEST_HIGHER, + ignore_existing=True) elif args.operation == 'list': do_list(args, app) elif args.operation == 'download': From 51324da24d42b47a80bf15f09dbfea9f35df4bab Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Mon, 20 Jul 2020 01:08:40 +0800 Subject: [PATCH 014/119] Allow -like arguments for the list operation. --- qubesadmin/tools/qvm_template.py | 45 +++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 9a68600..145f7c8 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -3,6 +3,7 @@ import argparse import datetime import enum +import fnmatch import os import shutil import subprocess @@ -262,7 +263,7 @@ def qrexec_repoquery(args, app, spec='*'): if not entry[0].startswith(PACKAGE_NAME_PREFIX): continue entry[0] = entry[0][len(PACKAGE_NAME_PREFIX):] - result.append(entry) + result.append(tuple(entry)) return result def qrexec_download(args, app, spec, path, dlsize=None): @@ -289,6 +290,28 @@ def qrexec_download(args, app, spec, path, dlsize=None): def build_version_str(evr): return '%s:%s-%s' % evr +def is_match_spec(name, epoch, version, release, spec): + # Refer to "NEVRA Matching" in the DNF documentation + # NOTE: Currently "arch" is ignored as the templates should be of "noarch" + if epoch != 0: + targets = [ + f'{name}-{epoch}:{version}-{release}', + f'{name}', + f'{name}-{epoch}:{version}' + ] + else: + targets = [ + f'{name}-{epoch}:{version}-{release}', + f'{name}-{version}-{release}', + f'{name}', + f'{name}-{epoch}:{version}', + f'{name}-{version}' + ] + for prio, target in enumerate(targets): + if fnmatch.fnmatch(target, spec): + return True, prio + return False, float('inf') + def do_list(args, app): # TODO: Check local template name actually matches to account for renames # TODO: Also display repo like `dnf list` @@ -298,7 +321,13 @@ def do_list(args, app): args.all = True if args.all or args.available or args.extras or args.upgrades: - query_res = qrexec_repoquery(args, app) + if args.templates: + query_res = set() + for spec in args.templates: + query_res |= set(qrexec_repoquery(args, app, spec)) + query_res = list(query_res) + else: + query_res = qrexec_repoquery(args, app) if args.installed or args.all: for vm in app.domains: @@ -307,8 +336,16 @@ def do_list(args, app): vm.features['template-epoch'], vm.features['template-version'], vm.features['template-release'])) - tpl_list.append( - (vm.name, version_str, TemplateState.INSTALLED.value)) + if not args.templates or \ + any(is_match_spec( + vm.name, + vm.features['template-epoch'], + vm.features['template-version'], + vm.features['template-release'], + spec)[0] + for spec in args.templates): + tpl_list.append( + (vm.name, version_str, TemplateState.INSTALLED.value)) if args.available or args.all: #pylint: disable=unused-variable From d656554822a89fa44a265912b427cc428953441a Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 21 Jul 2020 02:04:55 +0800 Subject: [PATCH 015/119] Initial implementation for "qvm-template info". --- qubesadmin/tools/qvm_template.py | 153 +++++++++++++++++++++---------- 1 file changed, 105 insertions(+), 48 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 145f7c8..facf884 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -4,6 +4,7 @@ import argparse import datetime import enum import fnmatch +import itertools import os import shutil import subprocess @@ -68,6 +69,9 @@ parser.add_argument('--installed', action='store_true') parser.add_argument('--available', action='store_true') parser.add_argument('--extras', action='store_true') parser.add_argument('--upgrades', action='store_true') +# qvm-template search +# Already defined above +#parser.add_argument('--all', action='store_true') class TemplateState(enum.Enum): INSTALLED = 'installed' @@ -75,6 +79,15 @@ class TemplateState(enum.Enum): EXTRA = 'extra' UPGRADABLE = 'upgradable' + def title(self): + TEMPLATE_TITLES = { + TemplateState.INSTALLED: 'Installed Templates', + TemplateState.AVAILABLE: 'Available Templates', + TemplateState.EXTRA: 'Extra Templates', + TemplateState.UPGRADABLE: 'Available Upgrades' + } + return TEMPLATE_TITLES[self] + class VersionSelector(enum.Enum): LATEST = enum.auto() REINSTALL = enum.auto() @@ -134,14 +147,15 @@ def install(args, app, version_selector=VersionSelector.LATEST, if template.endswith('.rpm'): if not os.path.exists(template): parser.error('RPM file \'%s\' not found.' % template) - rpm_list.append(template) + size = os.path.getsize(template) + rpm_list.append((template, size, '@commandline')) os.makedirs(args.cachedir, exist_ok=True) dl_list = get_dl_list(args, app, version_selector=version_selector) dl_list_copy = dl_list.copy() # Verify that the templates are not yet installed - for name, (ver, _) in dl_list.items(): + for name, (ver, dlsize, reponame) in dl_list.items(): if not ignore_existing and name in app.domains: print(('Template \'%s\' already installed, skipping...' ' (You may want to use the {reinstall,upgrade,downgrade}' @@ -151,20 +165,26 @@ def install(args, app, version_selector=VersionSelector.LATEST, version_str = build_version_str(ver) target_file = \ '%s%s-%s.rpm' % (PACKAGE_NAME_PREFIX, name, version_str) - rpm_list.append(os.path.join(args.cachedir, target_file)) + rpm_list.append((os.path.join(args.cachedir, target_file), + dlsize, reponame)) dl_list = dl_list_copy download(args, app, path_override=args.cachedir, dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, version_selector=version_selector) - for rpmfile in rpm_list: - path = rpmfile + UNVERIFIED_SUFFIX + # XXX: Verify if package name is what we want? + for rpmfile, dlsize, reponame in rpm_list: + if reponame != '@commandline': + path = rpmfile + UNVERIFIED_SUFFIX + else: + path = rpmfile if not verify_rpm(path, args.nogpgcheck, transaction_set): parser.error('Package \'%s\' verification failed.' % rpmfile) - os.rename(path, rpmfile) + if reponame != '@commandline': + os.rename(path, rpmfile) - for rpmfile in rpm_list: + for rpmfile, dlsize, reponame in rpm_list: with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: package_hdr = get_package_hdr(rpmfile) package_name = package_hdr[rpm.RPMTAG_NAME] @@ -207,7 +227,9 @@ def install(args, app, version_selector=VersionSelector.LATEST, tpl.features['template-install-date'] = \ str(datetime.datetime.today()) tpl.features['template-name'] = name - # TODO: Store source repo + tpl.features['template-reponame'] = reponame + tpl.features['template-summary'] = \ + package_hdr[rpm.RPMTAG_SUMMARY] def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): if args.updatevm: @@ -312,11 +334,44 @@ def is_match_spec(name, epoch, version, release, spec): return True, prio return False, float('inf') -def do_list(args, app): - # TODO: Check local template name actually matches to account for renames - # TODO: Also display repo like `dnf list` +def list_templates(args, app, operation): tpl_list = [] + def append_list(data, status): + name, epoch, version, release, reponame, dlsize, summary = data + version_str = build_version_str((epoch, version, release)) + tpl_list.append((status, name, version_str, reponame)) + + def append_info(data, status): + name, epoch, version, release, reponame, dlsize, summary = data + tpl_list.append((status, 'Name', ':', name)) + tpl_list.append((status, 'Epoch', ':', epoch)) + tpl_list.append((status, 'Version', ':', version)) + tpl_list.append((status, 'Release', ':', release)) + tpl_list.append((status, 'Size', ':', + qubesadmin.utils.size_to_human(int(dlsize)))) + tpl_list.append((status, 'Repository', ':', reponame)) + tpl_list.append((status, 'Summary', ':', summary)) + tpl_list.append((status, ' ', ' ', ' ')) # empty line + + if operation == 'list': + append = append_list + elif operation == 'info': + append = append_info + else: + assert False and 'Unknown operation' + + def append_vm(vm, status): + if vm.name == vm.features['template-name']: + append(( + vm.features['template-name'], + vm.features['template-epoch'], + vm.features['template-version'], + vm.features['template-release'], + vm.features['template-reponame'], + vm.get_disk_utilization(), + vm.features['template-summary']), status) + if not (args.installed or args.available or args.extras or args.upgrades): args.all = True @@ -332,27 +387,19 @@ def do_list(args, app): if args.installed or args.all: for vm in app.domains: if 'template-install-date' in vm.features: - version_str = build_version_str(( - vm.features['template-epoch'], - vm.features['template-version'], - vm.features['template-release'])) if not args.templates or \ any(is_match_spec( - vm.name, + vm.features['template-name'], vm.features['template-epoch'], vm.features['template-version'], vm.features['template-release'], spec)[0] for spec in args.templates): - tpl_list.append( - (vm.name, version_str, TemplateState.INSTALLED.value)) + append_vm(vm, TemplateState.INSTALLED) if args.available or args.all: - #pylint: disable=unused-variable - for name, epoch, version, release, reponame, dlsize, summary \ - in query_res: - version_str = build_version_str((epoch, version, release)) - tpl_list.append((name, version_str, TemplateState.AVAILABLE.value)) + for data in query_res: + append(data, TemplateState.AVAILABLE) if args.extras: remote = set() @@ -362,12 +409,7 @@ def do_list(args, app): for vm in app.domains: if 'template-name' in vm.features and \ vm.features['template-name'] not in remote: - version_str = build_version_str(( - vm.features['template-epoch'], - vm.features['template-version'], - vm.features['template-release'])) - tpl_list.append( - (vm.name, version_str, TemplateState.EXTRA.value)) + append_vm(vm, TemplateState.EXTRA) if args.upgrades: local = {} @@ -377,15 +419,21 @@ def do_list(args, app): vm.features['template-epoch'], vm.features['template-version'], vm.features['template-release']) - for name, epoch, version, release, reponame, dlsize, summary \ - in query_res: + for data in query_res: + name, epoch, version, release, reponame, dlsize, summary = data if name in local: if rpm.labelCompare(local[name], (epoch, version, release)) < 0: - version_str = build_version_str((epoch, version, release)) - tpl_list.append( - (name, version_str, TemplateState.UPGRADABLE.value)) + append(data, TemplateState.UPGRADABLE) - qubesadmin.tools.print_table(tpl_list) + if len(tpl_list) == 0: + parser.error('No matching templates to list') + + for k, g in itertools.groupby(tpl_list, lambda x: x[0]): + print(k.title()) + qubesadmin.tools.print_table(list(map(lambda x: x[1:], g))) + +def search(args, app): + raise NotImplementedError def get_dl_list(args, app, version_selector=VersionSelector.LATEST): full_candid = {} @@ -404,11 +452,12 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): #pylint: disable=unused-variable for name, epoch, version, release, reponame, dlsize, summary \ in query_res: + assert reponame != '@commandline' ver = (epoch, version, release) if version_selector == VersionSelector.LATEST: if name not in candid \ - or rpm.labelCompare(candid[name], ver) < 0: - candid[name] = (ver, int(dlsize)) + or rpm.labelCompare(candid[name][0], ver) < 0: + candid[name] = (ver, int(dlsize), reponame) elif version_selector == VersionSelector.REINSTALL: if name not in app.domains: parser.error("Template '%s' not installed." % name) @@ -418,7 +467,7 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): vm.features['template-version'], vm.features['template-release']) if rpm.labelCompare(ver, cur_ver) == 0: - candid[name] = (ver, int(dlsize)) + candid[name] = (ver, int(dlsize), reponame) elif version_selector in [VersionSelector.LATEST_LOWER, VersionSelector.LATEST_HIGHER]: if name not in app.domains: @@ -433,8 +482,8 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): else 1 if rpm.labelCompare(ver, cur_ver) == cmp_res: if name not in candid \ - or rpm.labelCompare(candid[name], ver) < 0: - candid[name] = (ver, int(dlsize)) + or rpm.labelCompare(candid[name][0], ver) < 0: + candid[name] = (ver, int(dlsize), reponame) if len(candid) == 0: if version_selector == VersionSelector.LATEST: @@ -448,13 +497,12 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): elif version_selector == VersionSelector.LATEST_HIGHER: parser.error('Higher version of template \'%s\' not found.' \ % template) - sys.exit(1) # Merge & choose the template with the highest version - for name, (ver, dlsize) in candid.items(): + for name, (ver, dlsize, reponame) in candid.items(): if name not in full_candid \ - or rpm.labelCompare(full_candid[name], ver) < 0: - full_candid[name] = (ver, dlsize) + or rpm.labelCompare(full_candid[name][0], ver) < 0: + full_candid[name] = (ver, dlsize, reponame) return candid @@ -464,7 +512,7 @@ def download(args, app, path_override=None, dl_list = get_dl_list(args, app, version_selector=version_selector) path = path_override if path_override is not None else args.downloaddir - for name, (ver, dlsize) in dl_list.items(): + for name, (ver, dlsize, reponame) in dl_list.items(): version_str = build_version_str(ver) spec = PACKAGE_NAME_PREFIX + name + '-' + version_str target = os.path.join(path, '%s.rpm' % spec) @@ -497,8 +545,13 @@ def download(args, app, path_override=None, def remove(args, app): _ = app # unused + # Remove 'remove' entry from the args... + operation_idx = sys.argv.index('remove') + argv = sys.argv[1:operation_idx] + sys.argv[operation_idx+1:] + + # ...then pass the args to qvm-remove # Use exec so stdio can be shared easily - os.execvp('qvm-remove', ['qvm-remove'] + args.templates) + os.execvp('qvm-remove', ['qvm-remove'] + argv) def clean(args, app): # TODO: More fine-grained options @@ -507,7 +560,7 @@ def clean(args, app): shutil.rmtree(args.cachedir) def main(args=None, app=None): - args = parser.parse_args(args) + args, _ = parser.parse_known_args(args) if app is None: app = qubesadmin.Qubes() @@ -524,7 +577,11 @@ def main(args=None, app=None): install(args, app, version_selector=VersionSelector.LATEST_HIGHER, ignore_existing=True) elif args.operation == 'list': - do_list(args, app) + list_templates(args, app, 'list') + elif args.operation == 'info': + list_templates(args, app, 'info') + elif args.operation == 'search': + search(args, app) elif args.operation == 'download': download(args, app) elif args.operation == 'remove': From c573faa9c02b743a6a9c414cc0dacf8955789485 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 21 Jul 2020 19:34:16 +0800 Subject: [PATCH 016/119] Initial implementation for "qvm-template search". --- qubesadmin/tools/qvm_template.py | 94 ++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index facf884..d318db1 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 import argparse +import collections import datetime import enum import fnmatch +import functools import itertools import os import shutil @@ -80,6 +82,7 @@ class TemplateState(enum.Enum): UPGRADABLE = 'upgradable' def title(self): + #pylint: disable=invalid-name TEMPLATE_TITLES = { TemplateState.INSTALLED: 'Installed Templates', TemplateState.AVAILABLE: 'Available Templates', @@ -273,6 +276,8 @@ def qrexec_payload(args, app, spec): return payload def qrexec_repoquery(args, app, spec='*'): + # TODO: Perhaps expose stderr for error messages? + # At least need to provide message that, e.g., repoid does not exist. proc = qrexec_popen(args, app, 'qubes.TemplateSearch') payload = qrexec_payload(args, app, spec) stdout, _ = proc.communicate(payload.encode('UTF-8')) @@ -338,6 +343,7 @@ def list_templates(args, app, operation): tpl_list = [] def append_list(data, status): + #pylint: disable=unused-variable name, epoch, version, release, reponame, dlsize, summary = data version_str = build_version_str((epoch, version, release)) tpl_list.append((status, name, version_str, reponame)) @@ -386,7 +392,7 @@ def list_templates(args, app, operation): if args.installed or args.all: for vm in app.domains: - if 'template-install-date' in vm.features: + if 'template-name' in vm.features: if not args.templates or \ any(is_match_spec( vm.features['template-name'], @@ -403,6 +409,7 @@ def list_templates(args, app, operation): if args.extras: remote = set() + #pylint: disable=unused-variable for name, epoch, version, release, reponame, dlsize, summary \ in query_res: remote.add(name) @@ -428,12 +435,88 @@ def list_templates(args, app, operation): if len(tpl_list) == 0: parser.error('No matching templates to list') - for k, g in itertools.groupby(tpl_list, lambda x: x[0]): + for k, grp in itertools.groupby(tpl_list, lambda x: x[0]): print(k.title()) - qubesadmin.tools.print_table(list(map(lambda x: x[1:], g))) + qubesadmin.tools.print_table(list(map(lambda x: x[1:], grp))) def search(args, app): - raise NotImplementedError + # Search in both installed and available templates + query_res = qrexec_repoquery(args, app) + for vm in app.domains: + if 'template-name' in vm.features: + query_res.append(( + vm.features['template-name'], + vm.features['template-epoch'], + vm.features['template-version'], + vm.features['template-release'], + vm.features['template-reponame'], + vm.get_disk_utilization(), + vm.features['template-summary'])) + + # Get latest version for each template + query_res_tmp = [] + for name, grp in itertools.groupby(query_res, lambda x: x[0]): + def compare(lhs, rhs): + return lhs if rpm.labelCompare(lhs[1:4], rhs[1:4]) < 0 else rhs + query_res_tmp.append(functools.reduce(compare, grp)) + query_res = query_res_tmp + + #pylint: disable=invalid-name + WEIGHT_NAME_EXACT = 1 << 3 + WEIGHT_NAME = 1 << 2 + WEIGHT_SUMMARY = 1 << 1 + + search_res = collections.defaultdict(int) + first = True + for keyword in args.templates: + local_res = collections.defaultdict(int) + #pylint: disable=unused-variable + for idx, (name, epoch, version, release, reponame, dlsize, summary) \ + in enumerate(query_res): + if fnmatch.fnmatch(name, '*' + keyword + '*'): + local_res[idx] += WEIGHT_NAME + if fnmatch.fnmatch(summary, '*' + keyword + '*'): + local_res[idx] += WEIGHT_SUMMARY + if keyword == name: + local_res[idx] += WEIGHT_NAME_EXACT + for key, val in local_res.items(): + if args.all or first or key in search_res: + search_res[key] += val + first = False + + def key_func(x): + # Order by weight DESC, name ASC + weight = x[1] + name = query_res[x[0]][0] + return (-weight, name) + + search_res = sorted(search_res.items(), key=key_func) + + def gen_header(idx, weight): + # FIXME: "Exactly Matched" is printed even if the summary is not + # exactly matching + # TODO: Print matching keywords + #pylint: disable=unused-variable + name, epoch, version, release, reponame, dlsize, summary = \ + query_res[idx] + keys = [] + if weight & WEIGHT_NAME: + keys.append('Name') + if weight & WEIGHT_SUMMARY: + keys.append('Summary') + match = 'Exactly Matched' if weight & WEIGHT_NAME_EXACT else 'Matched' + return ' & '.join(keys) + ' ' + match + + last_weight = -1 + for idx, weight in search_res: + if last_weight != weight: + last_weight = weight + # Print headers + # XXX: The style is different from that of DNF + print(gen_header(idx, weight)) + name, epoch, version, release, reponame, dlsize, summary = \ + query_res[idx] + print(name, ':', summary) def get_dl_list(args, app, version_selector=VersionSelector.LATEST): full_candid = {} @@ -513,6 +596,7 @@ def download(args, app, path_override=None, path = path_override if path_override is not None else args.downloaddir for name, (ver, dlsize, reponame) in dl_list.items(): + _ = reponame # unused version_str = build_version_str(ver) spec = PACKAGE_NAME_PREFIX + name + '-' + version_str target = os.path.join(path, '%s.rpm' % spec) @@ -543,7 +627,7 @@ def download(args, app, path_override=None, sys.exit(1) def remove(args, app): - _ = app # unused + _ = args, app # unused # Remove 'remove' entry from the args... operation_idx = sys.argv.index('remove') From e6392ba4ec766ffa5a5ba396042d7361e8695244 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 26 Jul 2020 02:08:46 +0800 Subject: [PATCH 017/119] Add lock-file functionality for qvm-template install. --- qubesadmin/tools/qvm_template.py | 172 +++++++++++++++++-------------- 1 file changed, 92 insertions(+), 80 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index d318db1..969ec8f 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -25,6 +25,7 @@ TEMP_DIR = '/var/tmp' PACKAGE_NAME_PREFIX = 'qubes-template-' CACHE_DIR = os.path.join(xdg.BaseDirectory.xdg_cache_home, 'qvm-template') UNVERIFIED_SUFFIX = '.unverified' +LOCK_FILE = '/var/tmp/qvm-template.lck' def qubes_release(): if os.path.exists('/usr/share/qubes/marker-vm'): @@ -142,97 +143,108 @@ def parse_config(path): def install(args, app, version_selector=VersionSelector.LATEST, ignore_existing=False): - # TODO: Lock, mentioned in the note above - transaction_set = rpm.TransactionSet() + try: + with open(LOCK_FILE, 'x') as _: + pass + except FileExistsError: + parser.error(('%s already exists.' + ' Perhaps another instance of qvm-template is running?') + % LOCK_FILE) - rpm_list = [] - for template in args.templates: - if template.endswith('.rpm'): - if not os.path.exists(template): - parser.error('RPM file \'%s\' not found.' % template) - size = os.path.getsize(template) - rpm_list.append((template, size, '@commandline')) + try: + transaction_set = rpm.TransactionSet() - os.makedirs(args.cachedir, exist_ok=True) + rpm_list = [] + for template in args.templates: + if template.endswith('.rpm'): + if not os.path.exists(template): + parser.error('RPM file \'%s\' not found.' % template) + size = os.path.getsize(template) + rpm_list.append((template, size, '@commandline')) - dl_list = get_dl_list(args, app, version_selector=version_selector) - dl_list_copy = dl_list.copy() - # Verify that the templates are not yet installed - for name, (ver, dlsize, reponame) in dl_list.items(): - if not ignore_existing and name in app.domains: - print(('Template \'%s\' already installed, skipping...' - ' (You may want to use the {reinstall,upgrade,downgrade}' - ' operations.)') % name, file=sys.stderr) - del dl_list_copy[name] - else: - version_str = build_version_str(ver) - target_file = \ - '%s%s-%s.rpm' % (PACKAGE_NAME_PREFIX, name, version_str) - rpm_list.append((os.path.join(args.cachedir, target_file), - dlsize, reponame)) - dl_list = dl_list_copy + os.makedirs(args.cachedir, exist_ok=True) - download(args, app, path_override=args.cachedir, - dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, - version_selector=version_selector) - - # XXX: Verify if package name is what we want? - for rpmfile, dlsize, reponame in rpm_list: - if reponame != '@commandline': - path = rpmfile + UNVERIFIED_SUFFIX - else: - path = rpmfile - if not verify_rpm(path, args.nogpgcheck, transaction_set): - parser.error('Package \'%s\' verification failed.' % rpmfile) - if reponame != '@commandline': - os.rename(path, rpmfile) - - for rpmfile, dlsize, reponame in rpm_list: - with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: - package_hdr = get_package_hdr(rpmfile) - package_name = package_hdr[rpm.RPMTAG_NAME] - if not package_name.startswith(PACKAGE_NAME_PREFIX): - parser.error( - 'Illegal package name for package \'%s\'.' % rpmfile) - # Remove prefix to get the real template name - name = package_name[len(PACKAGE_NAME_PREFIX):] - - # Another check for already-downloaded RPMs + dl_list = get_dl_list(args, app, version_selector=version_selector) + dl_list_copy = dl_list.copy() + # Verify that the templates are not yet installed + for name, (ver, dlsize, reponame) in dl_list.items(): if not ignore_existing and name in app.domains: print(('Template \'%s\' already installed, skipping...' ' (You may want to use the {reinstall,upgrade,downgrade}' ' operations.)') % name, file=sys.stderr) - continue + del dl_list_copy[name] + else: + version_str = build_version_str(ver) + target_file = \ + '%s%s-%s.rpm' % (PACKAGE_NAME_PREFIX, name, version_str) + rpm_list.append((os.path.join(args.cachedir, target_file), + dlsize, reponame)) + dl_list = dl_list_copy - print('Installing template \'%s\'...' % name, file=sys.stderr) - extract_rpm(name, rpmfile, target) - cmdline = [ - 'qvm-template-postprocess', - '--really', - '--no-installed-by-rpm', - ] - if args.allow_pv: - cmdline.append('--allow-pv') - subprocess.check_call(cmdline + [ - 'post-install', - name, - target + PATH_PREFIX + '/' + name]) + download(args, app, path_override=args.cachedir, + dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, + version_selector=version_selector) - app.domains.refresh_cache(force=True) - tpl = app.domains[name] + # XXX: Verify if package name is what we want? + for rpmfile, dlsize, reponame in rpm_list: + if reponame != '@commandline': + path = rpmfile + UNVERIFIED_SUFFIX + else: + path = rpmfile + if not verify_rpm(path, args.nogpgcheck, transaction_set): + parser.error('Package \'%s\' verification failed.' % rpmfile) + if reponame != '@commandline': + os.rename(path, rpmfile) - tpl.features['template-epoch'] = \ - package_hdr[rpm.RPMTAG_EPOCHNUM] - tpl.features['template-version'] = \ - package_hdr[rpm.RPMTAG_VERSION] - tpl.features['template-release'] = \ - package_hdr[rpm.RPMTAG_RELEASE] - tpl.features['template-install-date'] = \ - str(datetime.datetime.today()) - tpl.features['template-name'] = name - tpl.features['template-reponame'] = reponame - tpl.features['template-summary'] = \ - package_hdr[rpm.RPMTAG_SUMMARY] + for rpmfile, dlsize, reponame in rpm_list: + with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: + package_hdr = get_package_hdr(rpmfile) + package_name = package_hdr[rpm.RPMTAG_NAME] + if not package_name.startswith(PACKAGE_NAME_PREFIX): + parser.error( + 'Illegal package name for package \'%s\'.' % rpmfile) + # Remove prefix to get the real template name + name = package_name[len(PACKAGE_NAME_PREFIX):] + + # Another check for already-downloaded RPMs + if not ignore_existing and name in app.domains: + print(('Template \'%s\' already installed, skipping...' + ' (You may want to use the' + ' {reinstall,upgrade,downgrade}' + ' operations.)') % name, file=sys.stderr) + continue + + print('Installing template \'%s\'...' % name, file=sys.stderr) + extract_rpm(name, rpmfile, target) + cmdline = [ + 'qvm-template-postprocess', + '--really', + '--no-installed-by-rpm', + ] + if args.allow_pv: + cmdline.append('--allow-pv') + subprocess.check_call(cmdline + [ + 'post-install', + name, + target + PATH_PREFIX + '/' + name]) + + app.domains.refresh_cache(force=True) + tpl = app.domains[name] + + tpl.features['template-epoch'] = \ + package_hdr[rpm.RPMTAG_EPOCHNUM] + tpl.features['template-version'] = \ + package_hdr[rpm.RPMTAG_VERSION] + tpl.features['template-release'] = \ + package_hdr[rpm.RPMTAG_RELEASE] + tpl.features['template-install-date'] = \ + str(datetime.datetime.today()) + tpl.features['template-name'] = name + tpl.features['template-reponame'] = reponame + tpl.features['template-summary'] = \ + package_hdr[rpm.RPMTAG_SUMMARY] + finally: + os.remove(LOCK_FILE) def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): if args.updatevm: From 37a72ecebf53d434eb684ef4cfdfe5c902599147 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 26 Jul 2020 02:11:03 +0800 Subject: [PATCH 018/119] Print error messages if qubes.TemplateSearch fails. --- qubesadmin/tools/qvm_template.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 969ec8f..85d13e6 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -260,6 +260,7 @@ def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): stderr=subprocess.PIPE) def qrexec_payload(args, app, spec): + # TODO: Support for force-refresh _ = app # unused def check_newline(string, name): @@ -288,13 +289,13 @@ def qrexec_payload(args, app, spec): return payload def qrexec_repoquery(args, app, spec='*'): - # TODO: Perhaps expose stderr for error messages? - # At least need to provide message that, e.g., repoid does not exist. proc = qrexec_popen(args, app, 'qubes.TemplateSearch') payload = qrexec_payload(args, app, spec) - stdout, _ = proc.communicate(payload.encode('UTF-8')) + stdout, stderr = proc.communicate(payload.encode('UTF-8')) stdout = stdout.decode('ASCII') if proc.wait() != 0: + for line in stderr.decode('ASCII').rstrip().split('\n'): + print('[Qrexec] %s' % line, file=sys.stderr) raise ConnectionError("qrexec call 'qubes.TemplateSearch' failed.") result = [] for line in stdout.strip().split('\n'): From 5e76bdb5f18d114f4bc94550461d88ad26a13db0 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 26 Jul 2020 02:11:57 +0800 Subject: [PATCH 019/119] Revamp "qvm-template search" and finish TODOs. --- qubesadmin/tools/qvm_template.py | 70 +++++++++++++++++--------------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 85d13e6..bba58a2 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -479,54 +479,60 @@ def search(args, app): WEIGHT_NAME = 1 << 2 WEIGHT_SUMMARY = 1 << 1 - search_res = collections.defaultdict(int) - first = True + search_res = collections.defaultdict(list) for keyword in args.templates: - local_res = collections.defaultdict(int) #pylint: disable=unused-variable for idx, (name, epoch, version, release, reponame, dlsize, summary) \ in enumerate(query_res): if fnmatch.fnmatch(name, '*' + keyword + '*'): - local_res[idx] += WEIGHT_NAME + exact = keyword == name + weight = WEIGHT_NAME_EXACT if exact else WEIGHT_NAME + search_res[idx].append((weight, keyword, exact)) if fnmatch.fnmatch(summary, '*' + keyword + '*'): - local_res[idx] += WEIGHT_SUMMARY - if keyword == name: - local_res[idx] += WEIGHT_NAME_EXACT - for key, val in local_res.items(): - if args.all or first or key in search_res: - search_res[key] += val - first = False + search_res[idx].append( + (WEIGHT_SUMMARY, keyword, keyword == summary)) + + # TODO: Search in description and URL for --all? + # Requires changes to the qrexec call qubes.TemplateSearch + if not args.all: + keywords = set(args.templates) + idxs = list(search_res.keys()) + for idx in idxs: + if keywords != set(x[1] for x in search_res[idx]): + del search_res[idx] def key_func(x): - # Order by weight DESC, name ASC - weight = x[1] - name = query_res[x[0]][0] - return (-weight, name) + # ORDER BY weight DESC, list_of_needles ASC, name ASC + idx, needles = x + weight = sum(t[0] for t in needles) + name = query_res[idx][0] + return (-weight, needles, name) search_res = sorted(search_res.items(), key=key_func) - def gen_header(idx, weight): - # FIXME: "Exactly Matched" is printed even if the summary is not - # exactly matching - # TODO: Print matching keywords + def gen_header(idx, needles): #pylint: disable=unused-variable name, epoch, version, release, reponame, dlsize, summary = \ query_res[idx] - keys = [] - if weight & WEIGHT_NAME: - keys.append('Name') - if weight & WEIGHT_SUMMARY: - keys.append('Summary') - match = 'Exactly Matched' if weight & WEIGHT_NAME_EXACT else 'Matched' - return ' & '.join(keys) + ' ' + match + fields = [] + weight_types = set(x[0] for x in needles) + if WEIGHT_NAME in weight_types: + fields.append('Name') + if WEIGHT_SUMMARY in weight_types: + fields.append('Summary') + exact = all(x[-1] for x in needles) + match = 'Exactly Matched' if exact else 'Matched' + keywords = sorted(list(set(x[1] for x in needles))) + return ' & '.join(fields) + ' ' + match + ': ' + ', '.join(keywords) - last_weight = -1 - for idx, weight in search_res: - if last_weight != weight: - last_weight = weight - # Print headers + last_header = '' + for idx, needles in search_res: + # Print headers + cur_header = gen_header(idx, needles) + if last_header != cur_header: + last_header = cur_header # XXX: The style is different from that of DNF - print(gen_header(idx, weight)) + print('===', cur_header, '===') name, epoch, version, release, reponame, dlsize, summary = \ query_res[idx] print(name, ':', summary) From 421dd74dd2829c14ef4e7caa60a6bda8214aeba6 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Mon, 27 Jul 2020 01:36:49 +0800 Subject: [PATCH 020/119] Check number of fields for qubes.TemplateSearch output. --- qubesadmin/tools/qvm_template.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index bba58a2..61925b3 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -299,7 +299,12 @@ def qrexec_repoquery(args, app, spec='*'): raise ConnectionError("qrexec call 'qubes.TemplateSearch' failed.") result = [] for line in stdout.strip().split('\n'): - entry = line.split(':') + # Make sure that there are at most 7 fields + # As there may be colons in the summary + entry = line.split(':', 7 - 1) + # If there are not enough entries, raise an error + if len(entry) < 7: + raise ConnectionError("qrexec call 'qubes.TemplateSearch' failed.") if not entry[0].startswith(PACKAGE_NAME_PREFIX): continue entry[0] = entry[0][len(PACKAGE_NAME_PREFIX):] From 88ee572cacca49adc9d88f782c0f79d5e0f114bd Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Wed, 29 Jul 2020 18:36:02 +0800 Subject: [PATCH 021/119] qvm-template: Incorporate additional metadata in qubes.TemplateSearch. --- qubesadmin/tools/qvm_template.py | 194 ++++++++++++++++++++++--------- 1 file changed, 138 insertions(+), 56 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 61925b3..ca3818b 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -8,6 +8,7 @@ import fnmatch import functools import itertools import os +import re import shutil import subprocess import sys @@ -167,7 +168,10 @@ def install(args, app, version_selector=VersionSelector.LATEST, dl_list = get_dl_list(args, app, version_selector=version_selector) dl_list_copy = dl_list.copy() # Verify that the templates are not yet installed + # TODO: Check that installed versions satisfy + # the {reinstall,{up,down}grade} requirements for name, (ver, dlsize, reponame) in dl_list.items(): + assert reponame != '@commandline' if not ignore_existing and name in app.domains: print(('Template \'%s\' already installed, skipping...' ' (You may want to use the {reinstall,upgrade,downgrade}' @@ -185,7 +189,6 @@ def install(args, app, version_selector=VersionSelector.LATEST, dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, version_selector=version_selector) - # XXX: Verify if package name is what we want? for rpmfile, dlsize, reponame in rpm_list: if reponame != '@commandline': path = rpmfile + UNVERIFIED_SUFFIX @@ -231,18 +234,27 @@ def install(args, app, version_selector=VersionSelector.LATEST, app.domains.refresh_cache(force=True) tpl = app.domains[name] + tpl.features['template-name'] = name tpl.features['template-epoch'] = \ package_hdr[rpm.RPMTAG_EPOCHNUM] tpl.features['template-version'] = \ package_hdr[rpm.RPMTAG_VERSION] tpl.features['template-release'] = \ package_hdr[rpm.RPMTAG_RELEASE] - tpl.features['template-install-date'] = \ - str(datetime.datetime.today()) - tpl.features['template-name'] = name tpl.features['template-reponame'] = reponame + tpl.features['template-buildtime'] = \ + str(datetime.datetime.fromtimestamp( + int(package_hdr[rpm.RPMTAG_BUILDTIME]))) + tpl.features['template-install-time'] = \ + str(datetime.datetime.today()) + tpl.features['template-license'] = \ + package_hdr[rpm.RPMTAG_LICENSE] + tpl.features['template-url'] = \ + package_hdr[rpm.RPMTAG_URL] tpl.features['template-summary'] = \ package_hdr[rpm.RPMTAG_SUMMARY] + tpl.features['template-description'] = \ + package_hdr[rpm.RPMTAG_DESCRIPTION].replace('\n', '|') finally: os.remove(LOCK_FILE) @@ -297,18 +309,52 @@ def qrexec_repoquery(args, app, spec='*'): for line in stderr.decode('ASCII').rstrip().split('\n'): print('[Qrexec] %s' % line, file=sys.stderr) raise ConnectionError("qrexec call 'qubes.TemplateSearch' failed.") + name_re = re.compile(r'^[A-Za-z0-9._+\-]*$') + evr_re = re.compile(r'^[A-Za-z0-9._+~]*$') + date_re = re.compile(r'^\d+-\d+-\d+ \d+:\d+$') + license_re = re.compile(r'^[A-Za-z0-9._+\-()]*$') result = [] - for line in stdout.strip().split('\n'): - # Make sure that there are at most 7 fields - # As there may be colons in the summary - entry = line.split(':', 7 - 1) - # If there are not enough entries, raise an error - if len(entry) < 7: - raise ConnectionError("qrexec call 'qubes.TemplateSearch' failed.") - if not entry[0].startswith(PACKAGE_NAME_PREFIX): + for line in stdout.split('|\n'): + # Note that there's an empty entry at the end as .strip() is not used. + # This is because if .strip() is used, the .split() will not work. + if line == '': continue - entry[0] = entry[0][len(PACKAGE_NAME_PREFIX):] - result.append(tuple(entry)) + entry = line.split('|') + try: + # If there is an incorrect number of entries, raise an error + name, epoch, version, release, reponame, dlsize, \ + buildtime, license, url, summary, description = entry + + # Ignore packages that are not templates + if not name.startswith(PACKAGE_NAME_PREFIX): + continue + name = name[len(PACKAGE_NAME_PREFIX):] + + # Check that the values make sense + if not re.fullmatch(name_re, name): + raise ValueError + for val in [epoch, version, release]: + if not re.fullmatch(evr_re, val): + raise ValueError + if not re.fullmatch(name_re, reponame): + raise ValueError + dlsize = int(dlsize) + # First verify that the date does not look weird, then parse it + if not re.fullmatch(date_re, buildtime): + raise ValueError + buildtime = datetime.datetime.strptime(buildtime, '%Y-%m-%d %H:%M') + # XXX: Perhaps whitelist licenses directly? + if not re.fullmatch(license_re, license): + raise ValueError + # Check name actually matches spec + if not is_match_spec(name, epoch, version, release, spec): + continue + + result.append((name, epoch, version, release, reponame, dlsize, + buildtime, license, url, summary, description)) + except (TypeError, ValueError): + raise ConnectionError(("qrexec call 'qubes.TemplateSearch' failed:" + " unexpected data format.")) return result def qrexec_download(args, app, spec, path, dlsize=None): @@ -360,22 +406,35 @@ def is_match_spec(name, epoch, version, release, spec): def list_templates(args, app, operation): tpl_list = [] - def append_list(data, status): + def append_list(data, status, install_time=None): + _ = install_time # unused #pylint: disable=unused-variable - name, epoch, version, release, reponame, dlsize, summary = data + name, epoch, version, release, reponame, dlsize, \ + buildtime, license, url, summary, description = data version_str = build_version_str((epoch, version, release)) tpl_list.append((status, name, version_str, reponame)) - def append_info(data, status): - name, epoch, version, release, reponame, dlsize, summary = data + def append_info(data, status, install_time=None): + name, epoch, version, release, reponame, dlsize, \ + buildtime, license, url, summary, description = data tpl_list.append((status, 'Name', ':', name)) tpl_list.append((status, 'Epoch', ':', epoch)) tpl_list.append((status, 'Version', ':', version)) tpl_list.append((status, 'Release', ':', release)) tpl_list.append((status, 'Size', ':', - qubesadmin.utils.size_to_human(int(dlsize)))) + qubesadmin.utils.size_to_human(dlsize))) tpl_list.append((status, 'Repository', ':', reponame)) + tpl_list.append((status, 'Buildtime', ':', str(buildtime))) + if install_time: + tpl_list.append((status, 'Install time', ':', str(install_time))) + tpl_list.append((status, 'URL', ':', url)) + tpl_list.append((status, 'License', ':', license)) tpl_list.append((status, 'Summary', ':', summary)) + # Only show "Description" for the first line + title = 'Description' + for line in description.splitlines(): + tpl_list.append((status, title, ':', line)) + title = '' tpl_list.append((status, ' ', ' ', ' ')) # empty line if operation == 'list': @@ -388,13 +447,20 @@ def list_templates(args, app, operation): def append_vm(vm, status): if vm.name == vm.features['template-name']: append(( - vm.features['template-name'], - vm.features['template-epoch'], - vm.features['template-version'], - vm.features['template-release'], - vm.features['template-reponame'], - vm.get_disk_utilization(), - vm.features['template-summary']), status) + vm.features['template-name'], + vm.features['template-epoch'], + vm.features['template-version'], + vm.features['template-release'], + vm.features['template-reponame'], + vm.get_disk_utilization(), + vm.features['template-buildtime'], + vm.features['template-license'], + vm.features['template-url'], + vm.features['template-summary'], + vm.features['template-description'].replace('|', '\n') + ), + status, + vm.features['template-install-time']) if not (args.installed or args.available or args.extras or args.upgrades): args.all = True @@ -428,8 +494,8 @@ def list_templates(args, app, operation): if args.extras: remote = set() #pylint: disable=unused-variable - for name, epoch, version, release, reponame, dlsize, summary \ - in query_res: + for name, epoch, version, release, reponame, dlsize, \ + buildtime, license, url, summary, description in query_res: remote.add(name) for vm in app.domains: if 'template-name' in vm.features and \ @@ -445,7 +511,8 @@ def list_templates(args, app, operation): vm.features['template-version'], vm.features['template-release']) for data in query_res: - name, epoch, version, release, reponame, dlsize, summary = data + name, epoch, version, release, reponame, dlsize, \ + buildtime, license, url, summary, description = data if name in local: if rpm.labelCompare(local[name], (epoch, version, release)) < 0: append(data, TemplateState.UPGRADABLE) @@ -469,7 +536,12 @@ def search(args, app): vm.features['template-release'], vm.features['template-reponame'], vm.get_disk_utilization(), - vm.features['template-summary'])) + datetime.datetime.fromisoformat( + vm.features['template-buildtime']), + vm.features['template-license'], + vm.features['template-url'], + vm.features['template-summary'], + vm.features['template-description'])) # Get latest version for each template query_res_tmp = [] @@ -480,24 +552,36 @@ def search(args, app): query_res = query_res_tmp #pylint: disable=invalid-name - WEIGHT_NAME_EXACT = 1 << 3 - WEIGHT_NAME = 1 << 2 - WEIGHT_SUMMARY = 1 << 1 + WEIGHT_NAME_EXACT = 1 << 4 + WEIGHT_NAME = 1 << 3 + WEIGHT_SUMMARY = 1 << 2 + WEIGHT_DESCRIPTION = 1 << 1 + WEIGHT_URL = 1 << 0 + + WEIGHT_TO_FIELD = [ + (WEIGHT_NAME_EXACT, 'Name'), + (WEIGHT_NAME, 'Name'), + (WEIGHT_SUMMARY, 'Summary'), + (WEIGHT_DESCRIPTION, 'Description'), + (WEIGHT_URL, 'URL')] search_res = collections.defaultdict(list) for keyword in args.templates: #pylint: disable=unused-variable - for idx, (name, epoch, version, release, reponame, dlsize, summary) \ + for idx, (name, epoch, version, release, reponame, dlsize, \ + buildtime, license, url, summary, description) \ in enumerate(query_res): - if fnmatch.fnmatch(name, '*' + keyword + '*'): - exact = keyword == name - weight = WEIGHT_NAME_EXACT if exact else WEIGHT_NAME - search_res[idx].append((weight, keyword, exact)) - if fnmatch.fnmatch(summary, '*' + keyword + '*'): - search_res[idx].append( - (WEIGHT_SUMMARY, keyword, keyword == summary)) + needles = [(name, WEIGHT_NAME), (summary, WEIGHT_SUMMARY)] + if args.all: + needles += \ + [(description, WEIGHT_DESCRIPTION), (url, WEIGHT_URL)] + for key, weight in needles: + if fnmatch.fnmatch(key, '*' + keyword + '*'): + exact = keyword == key + if exact and weight == WEIGHT_NAME: + weight = WEIGHT_NAME_EXACT + search_res[idx].append((weight, keyword, exact)) - # TODO: Search in description and URL for --all? # Requires changes to the qrexec call qubes.TemplateSearch if not args.all: keywords = set(args.templates) @@ -517,14 +601,13 @@ def search(args, app): def gen_header(idx, needles): #pylint: disable=unused-variable - name, epoch, version, release, reponame, dlsize, summary = \ - query_res[idx] + name, epoch, version, release, reponame, dlsize, \ + buildtime, license, url, summary, description = query_res[idx] fields = [] weight_types = set(x[0] for x in needles) - if WEIGHT_NAME in weight_types: - fields.append('Name') - if WEIGHT_SUMMARY in weight_types: - fields.append('Summary') + for weight, field in WEIGHT_TO_FIELD: + if weight in weight_types: + fields.append(field) exact = all(x[-1] for x in needles) match = 'Exactly Matched' if exact else 'Matched' keywords = sorted(list(set(x[1] for x in needles))) @@ -538,8 +621,8 @@ def search(args, app): last_header = cur_header # XXX: The style is different from that of DNF print('===', cur_header, '===') - name, epoch, version, release, reponame, dlsize, summary = \ - query_res[idx] + name, epoch, version, release, reponame, dlsize, \ + buildtime, license, url, summary, description = query_res[idx] print(name, ':', summary) def get_dl_list(args, app, version_selector=VersionSelector.LATEST): @@ -557,14 +640,13 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): # We only select one package for each distinct package name #pylint: disable=unused-variable - for name, epoch, version, release, reponame, dlsize, summary \ - in query_res: - assert reponame != '@commandline' + for name, epoch, version, release, reponame, dlsize, \ + buildtime, license, url, summary, description in query_res: ver = (epoch, version, release) if version_selector == VersionSelector.LATEST: if name not in candid \ or rpm.labelCompare(candid[name][0], ver) < 0: - candid[name] = (ver, int(dlsize), reponame) + candid[name] = (ver, dlsize, reponame) elif version_selector == VersionSelector.REINSTALL: if name not in app.domains: parser.error("Template '%s' not installed." % name) @@ -574,7 +656,7 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): vm.features['template-version'], vm.features['template-release']) if rpm.labelCompare(ver, cur_ver) == 0: - candid[name] = (ver, int(dlsize), reponame) + candid[name] = (ver, dlsize, reponame) elif version_selector in [VersionSelector.LATEST_LOWER, VersionSelector.LATEST_HIGHER]: if name not in app.domains: @@ -590,7 +672,7 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): if rpm.labelCompare(ver, cur_ver) == cmp_res: if name not in candid \ or rpm.labelCompare(candid[name][0], ver) < 0: - candid[name] = (ver, int(dlsize), reponame) + candid[name] = (ver, dlsize, reponame) if len(candid) == 0: if version_selector == VersionSelector.LATEST: From 8aa9ab9e89e7cc063be77000cbba7e23ba4e9891 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Wed, 29 Jul 2020 19:31:25 +0800 Subject: [PATCH 022/119] qvm-template: Remove downloaded file if the download is interrupted. --- qubesadmin/tools/qvm_template.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index ca3818b..1c8ca61 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -724,12 +724,16 @@ def download(args, app, path_override=None, done = True break except ConnectionError: + os.remove(target_suffix) if attempt + 1 < args.retries: print('\'%s\' download failed, retrying...' % spec, file=sys.stderr) + except: + # Also remove file if interrupted by other means + os.remove(target_suffix) + raise if not done: print('\'%s\' download failed.' % spec, file=sys.stderr) - os.remove(target_suffix) sys.exit(1) def remove(args, app): From f960ed4726e93b1faa04dc08a5e65bd933f9227d Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Wed, 29 Jul 2020 19:47:48 +0800 Subject: [PATCH 023/119] qvm-template: Add --refresh option and allow DNF cache to be used. --- qubesadmin/tools/qvm_template.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 1c8ca61..ea130ac 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -55,6 +55,8 @@ parser.add_argument('--repoid', action='append', help='Enable just specific repositories.') parser.add_argument('--releasever', default=qubes_release(), help='Override distro release version.') +parser.add_argument('--refresh', action='store_true', + help='Set repository metadata as expired before running the command.') parser.add_argument('--cachedir', default=CACHE_DIR, help='Override cache directory.') # qvm-template install @@ -271,7 +273,7 @@ def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): stdout=stdout, stderr=subprocess.PIPE) -def qrexec_payload(args, app, spec): +def qrexec_payload(args, app, spec, refresh): # TODO: Support for force-refresh _ = app # unused @@ -290,6 +292,8 @@ def qrexec_payload(args, app, spec): for repo in args.repoid if args.repoid else []: check_newline(repo, '--repoid') payload += '--repoid=%s\n' % repo + if refresh: + payload += '--refresh\n' check_newline(args.releasever, '--releasever') payload += '--releasever=%s\n' % args.releasever check_newline(spec, 'template name') @@ -300,9 +304,9 @@ def qrexec_payload(args, app, spec): payload += fd.read() + '\n' return payload -def qrexec_repoquery(args, app, spec='*'): +def qrexec_repoquery(args, app, spec='*', refresh=False): proc = qrexec_popen(args, app, 'qubes.TemplateSearch') - payload = qrexec_payload(args, app, spec) + payload = qrexec_payload(args, app, spec, refresh) stdout, stderr = proc.communicate(payload.encode('UTF-8')) stdout = stdout.decode('ASCII') if proc.wait() != 0: @@ -357,12 +361,12 @@ def qrexec_repoquery(args, app, spec='*'): " unexpected data format.")) return result -def qrexec_download(args, app, spec, path, dlsize=None): +def qrexec_download(args, app, spec, path, dlsize=None, refresh=False): with open(path, 'wb') as fd: # Don't filter ESCs for binary files proc = qrexec_popen(args, app, 'qubes.TemplateDownload', stdout=fd, filter_esc=False) - payload = qrexec_payload(args, app, spec) + payload = qrexec_payload(args, app, spec, refresh) proc.stdin.write(payload.encode('UTF-8')) proc.stdin.close() with tqdm.tqdm(desc=spec, total=dlsize, unit_scale=True, @@ -759,6 +763,9 @@ def main(args=None, app=None): if app is None: app = qubesadmin.Qubes() + if args.refresh: + qrexec_repoquery(args, app, refresh=True) + if args.operation == 'install': install(args, app) elif args.operation == 'reinstall': From ef59a658f44bc012aef9c886b67fd15bf4baddbc Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Wed, 29 Jul 2020 19:58:19 +0800 Subject: [PATCH 024/119] qvm-template: Make pylint happy by changing "license" to "licence". --- qubesadmin/tools/qvm_template.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index ea130ac..b988efc 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -316,7 +316,7 @@ def qrexec_repoquery(args, app, spec='*', refresh=False): name_re = re.compile(r'^[A-Za-z0-9._+\-]*$') evr_re = re.compile(r'^[A-Za-z0-9._+~]*$') date_re = re.compile(r'^\d+-\d+-\d+ \d+:\d+$') - license_re = re.compile(r'^[A-Za-z0-9._+\-()]*$') + licence_re = re.compile(r'^[A-Za-z0-9._+\-()]*$') result = [] for line in stdout.split('|\n'): # Note that there's an empty entry at the end as .strip() is not used. @@ -327,7 +327,7 @@ def qrexec_repoquery(args, app, spec='*', refresh=False): try: # If there is an incorrect number of entries, raise an error name, epoch, version, release, reponame, dlsize, \ - buildtime, license, url, summary, description = entry + buildtime, licence, url, summary, description = entry # Ignore packages that are not templates if not name.startswith(PACKAGE_NAME_PREFIX): @@ -348,14 +348,14 @@ def qrexec_repoquery(args, app, spec='*', refresh=False): raise ValueError buildtime = datetime.datetime.strptime(buildtime, '%Y-%m-%d %H:%M') # XXX: Perhaps whitelist licenses directly? - if not re.fullmatch(license_re, license): + if not re.fullmatch(licence_re, licence): raise ValueError # Check name actually matches spec if not is_match_spec(name, epoch, version, release, spec): continue result.append((name, epoch, version, release, reponame, dlsize, - buildtime, license, url, summary, description)) + buildtime, licence, url, summary, description)) except (TypeError, ValueError): raise ConnectionError(("qrexec call 'qubes.TemplateSearch' failed:" " unexpected data format.")) @@ -414,13 +414,13 @@ def list_templates(args, app, operation): _ = install_time # unused #pylint: disable=unused-variable name, epoch, version, release, reponame, dlsize, \ - buildtime, license, url, summary, description = data + buildtime, licence, url, summary, description = data version_str = build_version_str((epoch, version, release)) tpl_list.append((status, name, version_str, reponame)) def append_info(data, status, install_time=None): name, epoch, version, release, reponame, dlsize, \ - buildtime, license, url, summary, description = data + buildtime, licence, url, summary, description = data tpl_list.append((status, 'Name', ':', name)) tpl_list.append((status, 'Epoch', ':', epoch)) tpl_list.append((status, 'Version', ':', version)) @@ -432,7 +432,7 @@ def list_templates(args, app, operation): if install_time: tpl_list.append((status, 'Install time', ':', str(install_time))) tpl_list.append((status, 'URL', ':', url)) - tpl_list.append((status, 'License', ':', license)) + tpl_list.append((status, 'License', ':', licence)) tpl_list.append((status, 'Summary', ':', summary)) # Only show "Description" for the first line title = 'Description' @@ -499,7 +499,7 @@ def list_templates(args, app, operation): remote = set() #pylint: disable=unused-variable for name, epoch, version, release, reponame, dlsize, \ - buildtime, license, url, summary, description in query_res: + buildtime, licence, url, summary, description in query_res: remote.add(name) for vm in app.domains: if 'template-name' in vm.features and \ @@ -516,7 +516,7 @@ def list_templates(args, app, operation): vm.features['template-release']) for data in query_res: name, epoch, version, release, reponame, dlsize, \ - buildtime, license, url, summary, description = data + buildtime, licence, url, summary, description = data if name in local: if rpm.labelCompare(local[name], (epoch, version, release)) < 0: append(data, TemplateState.UPGRADABLE) @@ -573,7 +573,7 @@ def search(args, app): for keyword in args.templates: #pylint: disable=unused-variable for idx, (name, epoch, version, release, reponame, dlsize, \ - buildtime, license, url, summary, description) \ + buildtime, licence, url, summary, description) \ in enumerate(query_res): needles = [(name, WEIGHT_NAME), (summary, WEIGHT_SUMMARY)] if args.all: @@ -606,7 +606,7 @@ def search(args, app): def gen_header(idx, needles): #pylint: disable=unused-variable name, epoch, version, release, reponame, dlsize, \ - buildtime, license, url, summary, description = query_res[idx] + buildtime, licence, url, summary, description = query_res[idx] fields = [] weight_types = set(x[0] for x in needles) for weight, field in WEIGHT_TO_FIELD: @@ -626,7 +626,7 @@ def search(args, app): # XXX: The style is different from that of DNF print('===', cur_header, '===') name, epoch, version, release, reponame, dlsize, \ - buildtime, license, url, summary, description = query_res[idx] + buildtime, licence, url, summary, description = query_res[idx] print(name, ':', summary) def get_dl_list(args, app, version_selector=VersionSelector.LATEST): @@ -645,7 +645,7 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): # We only select one package for each distinct package name #pylint: disable=unused-variable for name, epoch, version, release, reponame, dlsize, \ - buildtime, license, url, summary, description in query_res: + buildtime, licence, url, summary, description in query_res: ver = (epoch, version, release) if version_selector == VersionSelector.LATEST: if name not in candid \ From 90e4f65bea820864c7e765c3aa7b44e6076fc1c1 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Wed, 29 Jul 2020 20:55:56 +0800 Subject: [PATCH 025/119] qvm-template*: Add option to specify pool to store created VM. --- qubesadmin/tools/qvm_template.py | 4 ++++ qubesadmin/tools/qvm_template_postprocess.py | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index b988efc..d058d2e 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -64,6 +64,8 @@ parser.add_argument('--nogpgcheck', action='store_true', help='Disable signature checks.') parser.add_argument('--allow-pv', action='store_true', help='Allow setting virt_mode to pv in configuration file.') +parser.add_argument('--pool', + help='Specify pool to store created VMs in.') # qvm-template download parser.add_argument('--downloaddir', default='.', help='Override download directory.') @@ -228,6 +230,8 @@ def install(args, app, version_selector=VersionSelector.LATEST, ] if args.allow_pv: cmdline.append('--allow-pv') + if args.pool: + cmdline += ['--pool', args.pool] subprocess.check_call(cmdline + [ 'post-install', name, diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py index 568cfde..c6b51f7 100644 --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -53,6 +53,8 @@ parser.add_argument('--no-installed-by-rpm', action='store_true', help='Do not set installed_by_rpm') parser.add_argument('--allow-pv', action='store_true', help='Allow setting virt_mode to pv in configuration file.') +parser.add_argument('--pool', + help='Specify pool to store created VMs in.') parser.add_argument('action', choices=['post-install', 'pre-remove'], help='Action to perform') parser.add_argument('name', action='store', @@ -259,7 +261,8 @@ def post_install(args): vm = app.add_new_vm('TemplateVM', name=args.name, - label=qubesadmin.config.defaults['template_label']) + label=qubesadmin.config.defaults['template_label'], + pool=args.pool) vm_created = True vm.log.info('Importing data') From 3ada7af0ebc122ecb40650e1afc8de1f3e3fcec7 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Fri, 31 Jul 2020 01:27:40 +0800 Subject: [PATCH 026/119] qvm-template: {reinstall,{up,down}grade}: Better handling and checks for existing version. --- qubesadmin/tools/qvm_template.py | 65 ++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index d058d2e..b614557 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -147,7 +147,7 @@ def parse_config(path): return dict(line.rstrip('\n').split('=', 1) for line in fd) def install(args, app, version_selector=VersionSelector.LATEST, - ignore_existing=False): + override_existing=False): try: with open(LOCK_FILE, 'x') as _: pass @@ -172,11 +172,9 @@ def install(args, app, version_selector=VersionSelector.LATEST, dl_list = get_dl_list(args, app, version_selector=version_selector) dl_list_copy = dl_list.copy() # Verify that the templates are not yet installed - # TODO: Check that installed versions satisfy - # the {reinstall,{up,down}grade} requirements for name, (ver, dlsize, reponame) in dl_list.items(): assert reponame != '@commandline' - if not ignore_existing and name in app.domains: + if not override_existing and name in app.domains: print(('Template \'%s\' already installed, skipping...' ' (You may want to use the {reinstall,upgrade,downgrade}' ' operations.)') % name, file=sys.stderr) @@ -214,12 +212,45 @@ def install(args, app, version_selector=VersionSelector.LATEST, name = package_name[len(PACKAGE_NAME_PREFIX):] # Another check for already-downloaded RPMs - if not ignore_existing and name in app.domains: + if not override_existing and name in app.domains: print(('Template \'%s\' already installed, skipping...' ' (You may want to use the' ' {reinstall,upgrade,downgrade}' ' operations.)') % name, file=sys.stderr) continue + # Check if local versus candidate version is in line with the + # operation + elif override_existing: + if name not in app.domains: + parser.error( + "Template '%s' not already installed." % name) + vm = app.domains[name] + pkg_evr = ( + str(package_hdr[rpm.RPMTAG_EPOCHNUM]), + package_hdr[rpm.RPMTAG_VERSION], + package_hdr[rpm.RPMTAG_RELEASE]) + vm_evr = ( + vm.features['template-epoch'], + vm.features['template-version'], + vm.features['template-release']) + cmp_res = rpm.labelCompare(pkg_evr, vm_evr) + if version_selector == VersionSelector.REINSTALL \ + and cmp_res != 0: + parser.error( + 'Same version of template \'%s\' not found.' \ + % name) + elif version_selector == VersionSelector.LATEST_LOWER \ + and cmp_res != -1: + print(("Template '%s' of lower version" + " already installed, skipping..." % name), + file=sys.stderr) + continue + elif version_selector == VersionSelector.LATEST_HIGHER \ + and cmp_res != 1: + print(("Template '%s' of higher version" + " already installed, skipping..." % name), + file=sys.stderr) + continue print('Installing template \'%s\'...' % name, file=sys.stderr) extract_rpm(name, rpmfile, target) @@ -278,7 +309,6 @@ def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): stderr=subprocess.PIPE) def qrexec_payload(args, app, spec, refresh): - # TODO: Support for force-refresh _ = app # unused def check_newline(string, name): @@ -657,7 +687,7 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): candid[name] = (ver, dlsize, reponame) elif version_selector == VersionSelector.REINSTALL: if name not in app.domains: - parser.error("Template '%s' not installed." % name) + parser.error("Template '%s' not already installed." % name) vm = app.domains[name] cur_ver = ( vm.features['template-epoch'], @@ -668,7 +698,7 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): elif version_selector in [VersionSelector.LATEST_LOWER, VersionSelector.LATEST_HIGHER]: if name not in app.domains: - parser.error("Template '%s' not installed." % name) + parser.error("Template '%s' not already installed." % name) vm = app.domains[name] cur_ver = ( vm.features['template-epoch'], @@ -682,18 +712,23 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): or rpm.labelCompare(candid[name][0], ver) < 0: candid[name] = (ver, dlsize, reponame) + # XXX: As it's possible to include version information in `template` + # Perhaps the messages can be improved if len(candid) == 0: if version_selector == VersionSelector.LATEST: parser.error('Template \'%s\' not found.' % template) elif version_selector == VersionSelector.REINSTALL: parser.error('Same version of template \'%s\' not found.' \ % template) + # Copy behavior of DNF and do nothing if version not found elif version_selector == VersionSelector.LATEST_LOWER: - parser.error('Lower version of template \'%s\' not found.' \ - % template) + print(("Template '%s' of lowest version" + " already installed, skipping..." % template), + file=sys.stderr) elif version_selector == VersionSelector.LATEST_HIGHER: - parser.error('Higher version of template \'%s\' not found.' \ - % template) + print(("Template '%s' of highest version" + " already installed, skipping..." % template), + file=sys.stderr) # Merge & choose the template with the highest version for name, (ver, dlsize, reponame) in candid.items(): @@ -774,13 +809,13 @@ def main(args=None, app=None): install(args, app) elif args.operation == 'reinstall': install(args, app, version_selector=VersionSelector.REINSTALL, - ignore_existing=True) + override_existing=True) elif args.operation == 'downgrade': install(args, app, version_selector=VersionSelector.LATEST_LOWER, - ignore_existing=True) + override_existing=True) elif args.operation == 'upgrade': install(args, app, version_selector=VersionSelector.LATEST_HIGHER, - ignore_existing=True) + override_existing=True) elif args.operation == 'list': list_templates(args, app, 'list') elif args.operation == 'info': From 233e411c2f90a1087ed3f4538f6dbb11055fcb3c Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 1 Aug 2020 02:24:29 +0800 Subject: [PATCH 027/119] qvm-template: Switch to namedtuples and other slight cleanup. --- qubesadmin/tools/qvm_template.py | 261 +++++++++++++++---------------- 1 file changed, 130 insertions(+), 131 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index b614557..b7c9a4f 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -103,6 +103,50 @@ class VersionSelector(enum.Enum): LATEST_LOWER = enum.auto() LATEST_HIGHER = enum.auto() +Template = collections.namedtuple('Template', [ + 'name', + 'epoch', + 'version', + 'release', + 'reponame', + 'dlsize', + 'buildtime', + 'licence', + 'url', + 'summary', + 'description' +]) + +DlEntry = collections.namedtuple('DlEntry', [ + 'evr', + 'reponame', + 'dlsize' +]) + +def query_local(vm): + return Template( + vm.features['template-name'], + vm.features['template-epoch'], + vm.features['template-version'], + vm.features['template-release'], + vm.features['template-reponame'], + vm.get_disk_utilization(), + vm.features['template-buildtime'], + vm.features['template-license'], + vm.features['template-url'], + vm.features['template-summary'], + vm.features['template-description'].replace('|', '\n')) + +def query_local_evr(vm): + return ( + vm.features['template-epoch'], + vm.features['template-version'], + vm.features['template-release']) + +def is_managed_template(vm): + return 'template-name' in vm.features \ + and vm.name == vm.features['template-name'] + # NOTE: Verifying RPMs this way is prone to TOCTOU. This is okay for local # files, but may create problems if multiple instances of `qvm-template` are # downloading the same file, so a lock is needed in that case. @@ -159,7 +203,7 @@ def install(args, app, version_selector=VersionSelector.LATEST, try: transaction_set = rpm.TransactionSet() - rpm_list = [] + rpm_list = [] # rpmfile, dlsize, reponame for template in args.templates: if template.endswith('.rpm'): if not os.path.exists(template): @@ -172,25 +216,27 @@ def install(args, app, version_selector=VersionSelector.LATEST, dl_list = get_dl_list(args, app, version_selector=version_selector) dl_list_copy = dl_list.copy() # Verify that the templates are not yet installed - for name, (ver, dlsize, reponame) in dl_list.items(): - assert reponame != '@commandline' + for name, entry in dl_list.items(): + # Should be ensured by checks in repoquery + assert entry.reponame != '@commandline' if not override_existing and name in app.domains: print(('Template \'%s\' already installed, skipping...' ' (You may want to use the {reinstall,upgrade,downgrade}' ' operations.)') % name, file=sys.stderr) del dl_list_copy[name] else: - version_str = build_version_str(ver) + version_str = build_version_str(entry.evr) target_file = \ '%s%s-%s.rpm' % (PACKAGE_NAME_PREFIX, name, version_str) rpm_list.append((os.path.join(args.cachedir, target_file), - dlsize, reponame)) + entry.dlsize, entry.reponame)) dl_list = dl_list_copy download(args, app, path_override=args.cachedir, dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, version_selector=version_selector) + # Verify package and remove unverified suffix for rpmfile, dlsize, reponame in rpm_list: if reponame != '@commandline': path = rpmfile + UNVERIFIED_SUFFIX @@ -201,6 +247,7 @@ def install(args, app, version_selector=VersionSelector.LATEST, if reponame != '@commandline': os.rename(path, rpmfile) + # Unpack and install for rpmfile, dlsize, reponame in rpm_list: with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: package_hdr = get_package_hdr(rpmfile) @@ -229,10 +276,7 @@ def install(args, app, version_selector=VersionSelector.LATEST, str(package_hdr[rpm.RPMTAG_EPOCHNUM]), package_hdr[rpm.RPMTAG_VERSION], package_hdr[rpm.RPMTAG_RELEASE]) - vm_evr = ( - vm.features['template-epoch'], - vm.features['template-version'], - vm.features['template-release']) + vm_evr = query_local_evr(vm) cmp_res = rpm.labelCompare(pkg_evr, vm_evr) if version_selector == VersionSelector.REINSTALL \ and cmp_res != 0: @@ -311,6 +355,8 @@ def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): def qrexec_payload(args, app, spec, refresh): _ = app # unused + # TODO: Check that spec != '---' + def check_newline(string, name): if '\n' in string: parser.error(f"Malformed {name}:" + @@ -360,6 +406,8 @@ def qrexec_repoquery(args, app, spec='*', refresh=False): entry = line.split('|') try: # If there is an incorrect number of entries, raise an error + # Unpack manually instead of stuffing into `Template` right away + # so that it's easier to mutate stuff. name, epoch, version, release, reponame, dlsize, \ buildtime, licence, url, summary, description = entry @@ -388,8 +436,8 @@ def qrexec_repoquery(args, app, spec='*', refresh=False): if not is_match_spec(name, epoch, version, release, spec): continue - result.append((name, epoch, version, release, reponame, dlsize, - buildtime, licence, url, summary, description)) + result.append(Template(name, epoch, version, release, reponame, + dlsize, buildtime, licence, url, summary, description)) except (TypeError, ValueError): raise ConnectionError(("qrexec call 'qubes.TemplateSearch' failed:" " unexpected data format.")) @@ -446,31 +494,27 @@ def list_templates(args, app, operation): def append_list(data, status, install_time=None): _ = install_time # unused - #pylint: disable=unused-variable - name, epoch, version, release, reponame, dlsize, \ - buildtime, licence, url, summary, description = data - version_str = build_version_str((epoch, version, release)) - tpl_list.append((status, name, version_str, reponame)) + version_str = build_version_str( + (data.epoch, data.version, data.release)) + tpl_list.append((status, data.name, version_str, data.reponame)) def append_info(data, status, install_time=None): - name, epoch, version, release, reponame, dlsize, \ - buildtime, licence, url, summary, description = data - tpl_list.append((status, 'Name', ':', name)) - tpl_list.append((status, 'Epoch', ':', epoch)) - tpl_list.append((status, 'Version', ':', version)) - tpl_list.append((status, 'Release', ':', release)) + tpl_list.append((status, 'Name', ':', data.name)) + tpl_list.append((status, 'Epoch', ':', data.epoch)) + tpl_list.append((status, 'Version', ':', data.version)) + tpl_list.append((status, 'Release', ':', data.release)) tpl_list.append((status, 'Size', ':', - qubesadmin.utils.size_to_human(dlsize))) - tpl_list.append((status, 'Repository', ':', reponame)) - tpl_list.append((status, 'Buildtime', ':', str(buildtime))) + qubesadmin.utils.size_to_human(data.dlsize))) + tpl_list.append((status, 'Repository', ':', data.reponame)) + tpl_list.append((status, 'Buildtime', ':', str(data.buildtime))) if install_time: tpl_list.append((status, 'Install time', ':', str(install_time))) - tpl_list.append((status, 'URL', ':', url)) - tpl_list.append((status, 'License', ':', licence)) - tpl_list.append((status, 'Summary', ':', summary)) + tpl_list.append((status, 'URL', ':', data.url)) + tpl_list.append((status, 'License', ':', data.licence)) + tpl_list.append((status, 'Summary', ':', data.summary)) # Only show "Description" for the first line title = 'Description' - for line in description.splitlines(): + for line in data.description.splitlines(): tpl_list.append((status, title, ':', line)) title = '' tpl_list.append((status, ' ', ' ', ' ')) # empty line @@ -483,22 +527,7 @@ def list_templates(args, app, operation): assert False and 'Unknown operation' def append_vm(vm, status): - if vm.name == vm.features['template-name']: - append(( - vm.features['template-name'], - vm.features['template-epoch'], - vm.features['template-version'], - vm.features['template-release'], - vm.features['template-reponame'], - vm.get_disk_utilization(), - vm.features['template-buildtime'], - vm.features['template-license'], - vm.features['template-url'], - vm.features['template-summary'], - vm.features['template-description'].replace('|', '\n') - ), - status, - vm.features['template-install-time']) + append(query_local(vm), status, vm.features['template-install-time']) if not (args.installed or args.available or args.extras or args.upgrades): args.all = True @@ -514,15 +543,13 @@ def list_templates(args, app, operation): if args.installed or args.all: for vm in app.domains: - if 'template-name' in vm.features: + if is_managed_template(vm): if not args.templates or \ any(is_match_spec( - vm.features['template-name'], - vm.features['template-epoch'], - vm.features['template-version'], - vm.features['template-release'], - spec)[0] - for spec in args.templates): + vm.features['template-name'], + *query_local_evr(vm), + spec)[0] + for spec in args.templates): append_vm(vm, TemplateState.INSTALLED) if args.available or args.all: @@ -531,29 +558,23 @@ def list_templates(args, app, operation): if args.extras: remote = set() - #pylint: disable=unused-variable - for name, epoch, version, release, reponame, dlsize, \ - buildtime, licence, url, summary, description in query_res: - remote.add(name) + for data in query_res: + remote.add(data.name) for vm in app.domains: - if 'template-name' in vm.features and \ + if is_managed_template(vm) and \ vm.features['template-name'] not in remote: append_vm(vm, TemplateState.EXTRA) if args.upgrades: local = {} for vm in app.domains: - if 'template-name' in vm.features: - local[vm.features['template-name']] = ( - vm.features['template-epoch'], - vm.features['template-version'], - vm.features['template-release']) - for data in query_res: - name, epoch, version, release, reponame, dlsize, \ - buildtime, licence, url, summary, description = data - if name in local: - if rpm.labelCompare(local[name], (epoch, version, release)) < 0: - append(data, TemplateState.UPGRADABLE) + if is_managed_template(vm): + local[vm.features['template-name']] = query_local_evr(vm) + for entry in query_res: + if entry.name in local: + if rpm.labelCompare(local[entry.name], + (entry.epoch, entry.version, entry.release)) < 0: + append(entry, TemplateState.UPGRADABLE) if len(tpl_list) == 0: parser.error('No matching templates to list') @@ -566,24 +587,12 @@ def search(args, app): # Search in both installed and available templates query_res = qrexec_repoquery(args, app) for vm in app.domains: - if 'template-name' in vm.features: - query_res.append(( - vm.features['template-name'], - vm.features['template-epoch'], - vm.features['template-version'], - vm.features['template-release'], - vm.features['template-reponame'], - vm.get_disk_utilization(), - datetime.datetime.fromisoformat( - vm.features['template-buildtime']), - vm.features['template-license'], - vm.features['template-url'], - vm.features['template-summary'], - vm.features['template-description'])) + if is_managed_template(vm): + query_res.append(query_local(vm)) # Get latest version for each template query_res_tmp = [] - for name, grp in itertools.groupby(query_res, lambda x: x[0]): + for name, grp in itertools.groupby(sorted(query_res), lambda x: x[0]): def compare(lhs, rhs): return lhs if rpm.labelCompare(lhs[1:4], rhs[1:4]) < 0 else rhs query_res_tmp.append(functools.reduce(compare, grp)) @@ -605,14 +614,12 @@ def search(args, app): search_res = collections.defaultdict(list) for keyword in args.templates: - #pylint: disable=unused-variable - for idx, (name, epoch, version, release, reponame, dlsize, \ - buildtime, licence, url, summary, description) \ - in enumerate(query_res): - needles = [(name, WEIGHT_NAME), (summary, WEIGHT_SUMMARY)] + for idx, entry in enumerate(query_res): + needles = \ + [(entry.name, WEIGHT_NAME), (entry.summary, WEIGHT_SUMMARY)] if args.all: - needles += \ - [(description, WEIGHT_DESCRIPTION), (url, WEIGHT_URL)] + needles += [(entry.description, WEIGHT_DESCRIPTION), + (entry.url, WEIGHT_URL)] for key, weight in needles: if fnmatch.fnmatch(key, '*' + keyword + '*'): exact = keyword == key @@ -620,7 +627,6 @@ def search(args, app): weight = WEIGHT_NAME_EXACT search_res[idx].append((weight, keyword, exact)) - # Requires changes to the qrexec call qubes.TemplateSearch if not args.all: keywords = set(args.templates) idxs = list(search_res.keys()) @@ -638,9 +644,6 @@ def search(args, app): search_res = sorted(search_res.items(), key=key_func) def gen_header(idx, needles): - #pylint: disable=unused-variable - name, epoch, version, release, reponame, dlsize, \ - buildtime, licence, url, summary, description = query_res[idx] fields = [] weight_types = set(x[0] for x in needles) for weight, field in WEIGHT_TO_FIELD: @@ -659,9 +662,7 @@ def search(args, app): last_header = cur_header # XXX: The style is different from that of DNF print('===', cur_header, '===') - name, epoch, version, release, reponame, dlsize, \ - buildtime, licence, url, summary, description = query_res[idx] - print(name, ':', summary) + print(query_res[idx].name, ':', query_res[idx].summary) def get_dl_list(args, app, version_selector=VersionSelector.LATEST): full_candid = {} @@ -677,40 +678,38 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): query_res = qrexec_repoquery(args, app, PACKAGE_NAME_PREFIX + template) # We only select one package for each distinct package name - #pylint: disable=unused-variable - for name, epoch, version, release, reponame, dlsize, \ - buildtime, licence, url, summary, description in query_res: - ver = (epoch, version, release) + # TODO: Check local VM is managed by qvm-template + for entry in query_res: + ver = (entry.epoch, entry.version, entry.release) + insert = False if version_selector == VersionSelector.LATEST: - if name not in candid \ - or rpm.labelCompare(candid[name][0], ver) < 0: - candid[name] = (ver, dlsize, reponame) + if entry.name not in candid \ + or rpm.labelCompare(candid[entry.name][0], ver) < 0: + insert = True elif version_selector == VersionSelector.REINSTALL: - if name not in app.domains: - parser.error("Template '%s' not already installed." % name) - vm = app.domains[name] - cur_ver = ( - vm.features['template-epoch'], - vm.features['template-version'], - vm.features['template-release']) + if entry.name not in app.domains: + parser.error( + "Template '%s' not already installed." % entry.name) + vm = app.domains[entry.name] + cur_ver = query_local_evr(vm) if rpm.labelCompare(ver, cur_ver) == 0: - candid[name] = (ver, dlsize, reponame) + insert = True elif version_selector in [VersionSelector.LATEST_LOWER, VersionSelector.LATEST_HIGHER]: - if name not in app.domains: - parser.error("Template '%s' not already installed." % name) - vm = app.domains[name] - cur_ver = ( - vm.features['template-epoch'], - vm.features['template-version'], - vm.features['template-release']) + if entry.name not in app.domains: + parser.error( + "Template '%s' not already installed." % entry.name) + vm = app.domains[entry.name] + cur_ver = query_local_evr(vm) cmp_res = -1 \ - if version_selector == VersionSelector.LATEST_LOWER \ - else 1 + if version_selector == VersionSelector.LATEST_LOWER \ + else 1 if rpm.labelCompare(ver, cur_ver) == cmp_res: - if name not in candid \ - or rpm.labelCompare(candid[name][0], ver) < 0: - candid[name] = (ver, dlsize, reponame) + if entry.name not in candid \ + or rpm.labelCompare(candid[entry.name][0], ver) < 0: + insert = True + if insert: + candid[entry.name] = DlEntry(ver, entry.reponame, entry.dlsize) # XXX: As it's possible to include version information in `template` # Perhaps the messages can be improved @@ -731,10 +730,10 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): file=sys.stderr) # Merge & choose the template with the highest version - for name, (ver, dlsize, reponame) in candid.items(): + for name, entry in candid.items(): if name not in full_candid \ - or rpm.labelCompare(full_candid[name][0], ver) < 0: - full_candid[name] = (ver, dlsize, reponame) + or rpm.labelCompare(full_candid[name].evr, entry.evr) < 0: + full_candid[name] = entry return candid @@ -744,9 +743,8 @@ def download(args, app, path_override=None, dl_list = get_dl_list(args, app, version_selector=version_selector) path = path_override if path_override is not None else args.downloaddir - for name, (ver, dlsize, reponame) in dl_list.items(): - _ = reponame # unused - version_str = build_version_str(ver) + for name, entry in dl_list.items(): + version_str = build_version_str(entry.evr) spec = PACKAGE_NAME_PREFIX + name + '-' + version_str target = os.path.join(path, '%s.rpm' % spec) target_suffix = target + suffix @@ -763,7 +761,8 @@ def download(args, app, path_override=None, done = False for attempt in range(args.retries): try: - qrexec_download(args, app, spec, target_suffix, dlsize) + qrexec_download(args, app, spec, target_suffix, + entry.dlsize) done = True break except ConnectionError: From 3d0a39523b1f59f98f08ae3d7de5da7a63c81454 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 1 Aug 2020 02:40:27 +0800 Subject: [PATCH 028/119] qvm-template: Reorder functions. --- qubesadmin/tools/qvm_template.py | 542 +++++++++++++++---------------- 1 file changed, 269 insertions(+), 273 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index b7c9a4f..2dc605c 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -123,6 +123,31 @@ DlEntry = collections.namedtuple('DlEntry', [ 'dlsize' ]) +def build_version_str(evr): + return '%s:%s-%s' % evr + +def is_match_spec(name, epoch, version, release, spec): + # Refer to "NEVRA Matching" in the DNF documentation + # NOTE: Currently "arch" is ignored as the templates should be of "noarch" + if epoch != 0: + targets = [ + f'{name}-{epoch}:{version}-{release}', + f'{name}', + f'{name}-{epoch}:{version}' + ] + else: + targets = [ + f'{name}-{epoch}:{version}-{release}', + f'{name}-{version}-{release}', + f'{name}', + f'{name}-{epoch}:{version}', + f'{name}-{version}' + ] + for prio, target in enumerate(targets): + if fnmatch.fnmatch(target, spec): + return True, prio + return False, float('inf') + def query_local(vm): return Template( vm.features['template-name'], @@ -147,10 +172,135 @@ def is_managed_template(vm): return 'template-name' in vm.features \ and vm.name == vm.features['template-name'] -# NOTE: Verifying RPMs this way is prone to TOCTOU. This is okay for local -# files, but may create problems if multiple instances of `qvm-template` are -# downloading the same file, so a lock is needed in that case. +def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): + if args.updatevm: + return app.domains[args.updatevm].run_service( + service, + filter_esc=filter_esc, + stdout=stdout) + return subprocess.Popen([ + '/etc/qubes-rpc/%s' % service, + ], + stdin=subprocess.PIPE, + stdout=stdout, + stderr=subprocess.PIPE) + +def qrexec_payload(args, app, spec, refresh): + _ = app # unused + + # TODO: Check that spec != '---' + + def check_newline(string, name): + if '\n' in string: + parser.error(f"Malformed {name}:" + + " argument should not contain '\\n'.") + + payload = '' + for repo in args.enablerepo if args.enablerepo else []: + check_newline(repo, '--enablerepo') + payload += '--enablerepo=%s\n' % repo + for repo in args.disablerepo if args.disablerepo else []: + check_newline(repo, '--disablerepo') + payload += '--disablerepo=%s\n' % repo + for repo in args.repoid if args.repoid else []: + check_newline(repo, '--repoid') + payload += '--repoid=%s\n' % repo + if refresh: + payload += '--refresh\n' + check_newline(args.releasever, '--releasever') + payload += '--releasever=%s\n' % args.releasever + check_newline(spec, 'template name') + payload += spec + '\n' + payload += '---\n' + for path in args.repo_files: + with open(path, 'r') as fd: + payload += fd.read() + '\n' + return payload + +def qrexec_repoquery(args, app, spec='*', refresh=False): + proc = qrexec_popen(args, app, 'qubes.TemplateSearch') + payload = qrexec_payload(args, app, spec, refresh) + stdout, stderr = proc.communicate(payload.encode('UTF-8')) + stdout = stdout.decode('ASCII') + if proc.wait() != 0: + for line in stderr.decode('ASCII').rstrip().split('\n'): + print('[Qrexec] %s' % line, file=sys.stderr) + raise ConnectionError("qrexec call 'qubes.TemplateSearch' failed.") + name_re = re.compile(r'^[A-Za-z0-9._+\-]*$') + evr_re = re.compile(r'^[A-Za-z0-9._+~]*$') + date_re = re.compile(r'^\d+-\d+-\d+ \d+:\d+$') + licence_re = re.compile(r'^[A-Za-z0-9._+\-()]*$') + result = [] + for line in stdout.split('|\n'): + # Note that there's an empty entry at the end as .strip() is not used. + # This is because if .strip() is used, the .split() will not work. + if line == '': + continue + entry = line.split('|') + try: + # If there is an incorrect number of entries, raise an error + # Unpack manually instead of stuffing into `Template` right away + # so that it's easier to mutate stuff. + name, epoch, version, release, reponame, dlsize, \ + buildtime, licence, url, summary, description = entry + + # Ignore packages that are not templates + if not name.startswith(PACKAGE_NAME_PREFIX): + continue + name = name[len(PACKAGE_NAME_PREFIX):] + + # Check that the values make sense + if not re.fullmatch(name_re, name): + raise ValueError + for val in [epoch, version, release]: + if not re.fullmatch(evr_re, val): + raise ValueError + if not re.fullmatch(name_re, reponame): + raise ValueError + dlsize = int(dlsize) + # First verify that the date does not look weird, then parse it + if not re.fullmatch(date_re, buildtime): + raise ValueError + buildtime = datetime.datetime.strptime(buildtime, '%Y-%m-%d %H:%M') + # XXX: Perhaps whitelist licenses directly? + if not re.fullmatch(licence_re, licence): + raise ValueError + # Check name actually matches spec + if not is_match_spec(name, epoch, version, release, spec): + continue + + result.append(Template(name, epoch, version, release, reponame, + dlsize, buildtime, licence, url, summary, description)) + except (TypeError, ValueError): + raise ConnectionError(("qrexec call 'qubes.TemplateSearch' failed:" + " unexpected data format.")) + return result + +def qrexec_download(args, app, spec, path, dlsize=None, refresh=False): + with open(path, 'wb') as fd: + # Don't filter ESCs for binary files + proc = qrexec_popen(args, app, 'qubes.TemplateDownload', + stdout=fd, filter_esc=False) + payload = qrexec_payload(args, app, spec, refresh) + proc.stdin.write(payload.encode('UTF-8')) + proc.stdin.close() + with tqdm.tqdm(desc=spec, total=dlsize, unit_scale=True, + unit_divisor=1000, unit='B') as pbar: + last = 0 + while proc.poll() is None: + cur = fd.tell() + pbar.update(cur - last) + last = cur + time.sleep(0.1) + if proc.wait() != 0: + raise ConnectionError( + "qrexec call 'qubes.TemplateDownload' failed.") + return True + def verify_rpm(path, nogpgcheck=False, transaction_set=None): + # NOTE: Verifying RPMs this way is prone to TOCTOU. This is okay for local + # files, but may create problems if multiple instances of `qvm-template` + # are downloading the same file, so a lock is needed in that case. if transaction_set is None: transaction_set = rpm.TransactionSet() with open(path, 'rb') as fd: @@ -186,9 +336,119 @@ def extract_rpm(name, path, target): ], stdin=rpm2cpio.stdout, stdout=subprocess.DEVNULL) return rpm2cpio.wait() == 0 and cpio.wait() == 0 -def parse_config(path): - with open(path, 'r') as fd: - return dict(line.rstrip('\n').split('=', 1) for line in fd) +def get_dl_list(args, app, version_selector=VersionSelector.LATEST): + full_candid = {} + for template in args.templates: + # This will be merged into `full_candid` later. + # It is separated so that we can check whether it is empty. + candid = {} + + # Skip local RPMs + if template.endswith('.rpm'): + continue + + query_res = qrexec_repoquery(args, app, PACKAGE_NAME_PREFIX + template) + + # We only select one package for each distinct package name + # TODO: Check local VM is managed by qvm-template + for entry in query_res: + ver = (entry.epoch, entry.version, entry.release) + insert = False + if version_selector == VersionSelector.LATEST: + if entry.name not in candid \ + or rpm.labelCompare(candid[entry.name][0], ver) < 0: + insert = True + elif version_selector == VersionSelector.REINSTALL: + if entry.name not in app.domains: + parser.error( + "Template '%s' not already installed." % entry.name) + vm = app.domains[entry.name] + cur_ver = query_local_evr(vm) + if rpm.labelCompare(ver, cur_ver) == 0: + insert = True + elif version_selector in [VersionSelector.LATEST_LOWER, + VersionSelector.LATEST_HIGHER]: + if entry.name not in app.domains: + parser.error( + "Template '%s' not already installed." % entry.name) + vm = app.domains[entry.name] + cur_ver = query_local_evr(vm) + cmp_res = -1 \ + if version_selector == VersionSelector.LATEST_LOWER \ + else 1 + if rpm.labelCompare(ver, cur_ver) == cmp_res: + if entry.name not in candid \ + or rpm.labelCompare(candid[entry.name][0], ver) < 0: + insert = True + if insert: + candid[entry.name] = DlEntry(ver, entry.reponame, entry.dlsize) + + # XXX: As it's possible to include version information in `template` + # Perhaps the messages can be improved + if len(candid) == 0: + if version_selector == VersionSelector.LATEST: + parser.error('Template \'%s\' not found.' % template) + elif version_selector == VersionSelector.REINSTALL: + parser.error('Same version of template \'%s\' not found.' \ + % template) + # Copy behavior of DNF and do nothing if version not found + elif version_selector == VersionSelector.LATEST_LOWER: + print(("Template '%s' of lowest version" + " already installed, skipping..." % template), + file=sys.stderr) + elif version_selector == VersionSelector.LATEST_HIGHER: + print(("Template '%s' of highest version" + " already installed, skipping..." % template), + file=sys.stderr) + + # Merge & choose the template with the highest version + for name, entry in candid.items(): + if name not in full_candid \ + or rpm.labelCompare(full_candid[name].evr, entry.evr) < 0: + full_candid[name] = entry + + return candid + +def download(args, app, path_override=None, + dl_list=None, suffix='', version_selector=VersionSelector.LATEST): + if dl_list is None: + dl_list = get_dl_list(args, app, version_selector=version_selector) + + path = path_override if path_override is not None else args.downloaddir + for name, entry in dl_list.items(): + version_str = build_version_str(entry.evr) + spec = PACKAGE_NAME_PREFIX + name + '-' + version_str + target = os.path.join(path, '%s.rpm' % spec) + target_suffix = target + suffix + if suffix != '' and os.path.exists(target_suffix): + print('\'%s\' already exists, skipping...' % target, + file=sys.stderr) + if os.path.exists(target): + print('\'%s\' already exists, skipping...' % target, + file=sys.stderr) + if suffix != '': + os.rename(target, target_suffix) + else: + print('Downloading \'%s\'...' % spec, file=sys.stderr) + done = False + for attempt in range(args.retries): + try: + qrexec_download(args, app, spec, target_suffix, + entry.dlsize) + done = True + break + except ConnectionError: + os.remove(target_suffix) + if attempt + 1 < args.retries: + print('\'%s\' download failed, retrying...' % spec, + file=sys.stderr) + except: + # Also remove file if interrupted by other means + os.remove(target_suffix) + raise + if not done: + print('\'%s\' download failed.' % spec, file=sys.stderr) + sys.exit(1) def install(args, app, version_selector=VersionSelector.LATEST, override_existing=False): @@ -339,156 +599,6 @@ def install(args, app, version_selector=VersionSelector.LATEST, finally: os.remove(LOCK_FILE) -def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): - if args.updatevm: - return app.domains[args.updatevm].run_service( - service, - filter_esc=filter_esc, - stdout=stdout) - return subprocess.Popen([ - '/etc/qubes-rpc/%s' % service, - ], - stdin=subprocess.PIPE, - stdout=stdout, - stderr=subprocess.PIPE) - -def qrexec_payload(args, app, spec, refresh): - _ = app # unused - - # TODO: Check that spec != '---' - - def check_newline(string, name): - if '\n' in string: - parser.error(f"Malformed {name}:" + - " argument should not contain '\\n'.") - - payload = '' - for repo in args.enablerepo if args.enablerepo else []: - check_newline(repo, '--enablerepo') - payload += '--enablerepo=%s\n' % repo - for repo in args.disablerepo if args.disablerepo else []: - check_newline(repo, '--disablerepo') - payload += '--disablerepo=%s\n' % repo - for repo in args.repoid if args.repoid else []: - check_newline(repo, '--repoid') - payload += '--repoid=%s\n' % repo - if refresh: - payload += '--refresh\n' - check_newline(args.releasever, '--releasever') - payload += '--releasever=%s\n' % args.releasever - check_newline(spec, 'template name') - payload += spec + '\n' - payload += '---\n' - for path in args.repo_files: - with open(path, 'r') as fd: - payload += fd.read() + '\n' - return payload - -def qrexec_repoquery(args, app, spec='*', refresh=False): - proc = qrexec_popen(args, app, 'qubes.TemplateSearch') - payload = qrexec_payload(args, app, spec, refresh) - stdout, stderr = proc.communicate(payload.encode('UTF-8')) - stdout = stdout.decode('ASCII') - if proc.wait() != 0: - for line in stderr.decode('ASCII').rstrip().split('\n'): - print('[Qrexec] %s' % line, file=sys.stderr) - raise ConnectionError("qrexec call 'qubes.TemplateSearch' failed.") - name_re = re.compile(r'^[A-Za-z0-9._+\-]*$') - evr_re = re.compile(r'^[A-Za-z0-9._+~]*$') - date_re = re.compile(r'^\d+-\d+-\d+ \d+:\d+$') - licence_re = re.compile(r'^[A-Za-z0-9._+\-()]*$') - result = [] - for line in stdout.split('|\n'): - # Note that there's an empty entry at the end as .strip() is not used. - # This is because if .strip() is used, the .split() will not work. - if line == '': - continue - entry = line.split('|') - try: - # If there is an incorrect number of entries, raise an error - # Unpack manually instead of stuffing into `Template` right away - # so that it's easier to mutate stuff. - name, epoch, version, release, reponame, dlsize, \ - buildtime, licence, url, summary, description = entry - - # Ignore packages that are not templates - if not name.startswith(PACKAGE_NAME_PREFIX): - continue - name = name[len(PACKAGE_NAME_PREFIX):] - - # Check that the values make sense - if not re.fullmatch(name_re, name): - raise ValueError - for val in [epoch, version, release]: - if not re.fullmatch(evr_re, val): - raise ValueError - if not re.fullmatch(name_re, reponame): - raise ValueError - dlsize = int(dlsize) - # First verify that the date does not look weird, then parse it - if not re.fullmatch(date_re, buildtime): - raise ValueError - buildtime = datetime.datetime.strptime(buildtime, '%Y-%m-%d %H:%M') - # XXX: Perhaps whitelist licenses directly? - if not re.fullmatch(licence_re, licence): - raise ValueError - # Check name actually matches spec - if not is_match_spec(name, epoch, version, release, spec): - continue - - result.append(Template(name, epoch, version, release, reponame, - dlsize, buildtime, licence, url, summary, description)) - except (TypeError, ValueError): - raise ConnectionError(("qrexec call 'qubes.TemplateSearch' failed:" - " unexpected data format.")) - return result - -def qrexec_download(args, app, spec, path, dlsize=None, refresh=False): - with open(path, 'wb') as fd: - # Don't filter ESCs for binary files - proc = qrexec_popen(args, app, 'qubes.TemplateDownload', - stdout=fd, filter_esc=False) - payload = qrexec_payload(args, app, spec, refresh) - proc.stdin.write(payload.encode('UTF-8')) - proc.stdin.close() - with tqdm.tqdm(desc=spec, total=dlsize, unit_scale=True, - unit_divisor=1000, unit='B') as pbar: - last = 0 - while proc.poll() is None: - cur = fd.tell() - pbar.update(cur - last) - last = cur - time.sleep(0.1) - if proc.wait() != 0: - raise ConnectionError( - "qrexec call 'qubes.TemplateDownload' failed.") - return True - -def build_version_str(evr): - return '%s:%s-%s' % evr - -def is_match_spec(name, epoch, version, release, spec): - # Refer to "NEVRA Matching" in the DNF documentation - # NOTE: Currently "arch" is ignored as the templates should be of "noarch" - if epoch != 0: - targets = [ - f'{name}-{epoch}:{version}-{release}', - f'{name}', - f'{name}-{epoch}:{version}' - ] - else: - targets = [ - f'{name}-{epoch}:{version}-{release}', - f'{name}-{version}-{release}', - f'{name}', - f'{name}-{epoch}:{version}', - f'{name}-{version}' - ] - for prio, target in enumerate(targets): - if fnmatch.fnmatch(target, spec): - return True, prio - return False, float('inf') - def list_templates(args, app, operation): tpl_list = [] @@ -664,120 +774,6 @@ def search(args, app): print('===', cur_header, '===') print(query_res[idx].name, ':', query_res[idx].summary) -def get_dl_list(args, app, version_selector=VersionSelector.LATEST): - full_candid = {} - for template in args.templates: - # This will be merged into `full_candid` later. - # It is separated so that we can check whether it is empty. - candid = {} - - # Skip local RPMs - if template.endswith('.rpm'): - continue - - query_res = qrexec_repoquery(args, app, PACKAGE_NAME_PREFIX + template) - - # We only select one package for each distinct package name - # TODO: Check local VM is managed by qvm-template - for entry in query_res: - ver = (entry.epoch, entry.version, entry.release) - insert = False - if version_selector == VersionSelector.LATEST: - if entry.name not in candid \ - or rpm.labelCompare(candid[entry.name][0], ver) < 0: - insert = True - elif version_selector == VersionSelector.REINSTALL: - if entry.name not in app.domains: - parser.error( - "Template '%s' not already installed." % entry.name) - vm = app.domains[entry.name] - cur_ver = query_local_evr(vm) - if rpm.labelCompare(ver, cur_ver) == 0: - insert = True - elif version_selector in [VersionSelector.LATEST_LOWER, - VersionSelector.LATEST_HIGHER]: - if entry.name not in app.domains: - parser.error( - "Template '%s' not already installed." % entry.name) - vm = app.domains[entry.name] - cur_ver = query_local_evr(vm) - cmp_res = -1 \ - if version_selector == VersionSelector.LATEST_LOWER \ - else 1 - if rpm.labelCompare(ver, cur_ver) == cmp_res: - if entry.name not in candid \ - or rpm.labelCompare(candid[entry.name][0], ver) < 0: - insert = True - if insert: - candid[entry.name] = DlEntry(ver, entry.reponame, entry.dlsize) - - # XXX: As it's possible to include version information in `template` - # Perhaps the messages can be improved - if len(candid) == 0: - if version_selector == VersionSelector.LATEST: - parser.error('Template \'%s\' not found.' % template) - elif version_selector == VersionSelector.REINSTALL: - parser.error('Same version of template \'%s\' not found.' \ - % template) - # Copy behavior of DNF and do nothing if version not found - elif version_selector == VersionSelector.LATEST_LOWER: - print(("Template '%s' of lowest version" - " already installed, skipping..." % template), - file=sys.stderr) - elif version_selector == VersionSelector.LATEST_HIGHER: - print(("Template '%s' of highest version" - " already installed, skipping..." % template), - file=sys.stderr) - - # Merge & choose the template with the highest version - for name, entry in candid.items(): - if name not in full_candid \ - or rpm.labelCompare(full_candid[name].evr, entry.evr) < 0: - full_candid[name] = entry - - return candid - -def download(args, app, path_override=None, - dl_list=None, suffix='', version_selector=VersionSelector.LATEST): - if dl_list is None: - dl_list = get_dl_list(args, app, version_selector=version_selector) - - path = path_override if path_override is not None else args.downloaddir - for name, entry in dl_list.items(): - version_str = build_version_str(entry.evr) - spec = PACKAGE_NAME_PREFIX + name + '-' + version_str - target = os.path.join(path, '%s.rpm' % spec) - target_suffix = target + suffix - if suffix != '' and os.path.exists(target_suffix): - print('\'%s\' already exists, skipping...' % target, - file=sys.stderr) - if os.path.exists(target): - print('\'%s\' already exists, skipping...' % target, - file=sys.stderr) - if suffix != '': - os.rename(target, target_suffix) - else: - print('Downloading \'%s\'...' % spec, file=sys.stderr) - done = False - for attempt in range(args.retries): - try: - qrexec_download(args, app, spec, target_suffix, - entry.dlsize) - done = True - break - except ConnectionError: - os.remove(target_suffix) - if attempt + 1 < args.retries: - print('\'%s\' download failed, retrying...' % spec, - file=sys.stderr) - except: - # Also remove file if interrupted by other means - os.remove(target_suffix) - raise - if not done: - print('\'%s\' download failed.' % spec, file=sys.stderr) - sys.exit(1) - def remove(args, app): _ = args, app # unused @@ -804,7 +800,9 @@ def main(args=None, app=None): if args.refresh: qrexec_repoquery(args, app, refresh=True) - if args.operation == 'install': + if args.operation == 'download': + download(args, app) + elif args.operation == 'install': install(args, app) elif args.operation == 'reinstall': install(args, app, version_selector=VersionSelector.REINSTALL, @@ -821,8 +819,6 @@ def main(args=None, app=None): list_templates(args, app, 'info') elif args.operation == 'search': search(args, app) - elif args.operation == 'download': - download(args, app) elif args.operation == 'remove': remove(args, app) elif args.operation == 'clean': From 40e7304f176f1284b288527caae26856f04da9c4 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 1 Aug 2020 02:56:59 +0800 Subject: [PATCH 029/119] qvm-template: Make pylint happy. --- qubesadmin/tools/qvm_template.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 2dc605c..b66bea0 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -463,13 +463,12 @@ def install(args, app, version_selector=VersionSelector.LATEST, try: transaction_set = rpm.TransactionSet() - rpm_list = [] # rpmfile, dlsize, reponame + rpm_list = [] # rpmfile, reponame for template in args.templates: if template.endswith('.rpm'): if not os.path.exists(template): parser.error('RPM file \'%s\' not found.' % template) - size = os.path.getsize(template) - rpm_list.append((template, size, '@commandline')) + rpm_list.append((template, '@commandline')) os.makedirs(args.cachedir, exist_ok=True) @@ -488,8 +487,8 @@ def install(args, app, version_selector=VersionSelector.LATEST, version_str = build_version_str(entry.evr) target_file = \ '%s%s-%s.rpm' % (PACKAGE_NAME_PREFIX, name, version_str) - rpm_list.append((os.path.join(args.cachedir, target_file), - entry.dlsize, entry.reponame)) + rpm_list.append( + (os.path.join(args.cachedir, target_file), entry.reponame)) dl_list = dl_list_copy download(args, app, path_override=args.cachedir, @@ -497,7 +496,7 @@ def install(args, app, version_selector=VersionSelector.LATEST, version_selector=version_selector) # Verify package and remove unverified suffix - for rpmfile, dlsize, reponame in rpm_list: + for rpmfile, reponame in rpm_list: if reponame != '@commandline': path = rpmfile + UNVERIFIED_SUFFIX else: @@ -508,7 +507,7 @@ def install(args, app, version_selector=VersionSelector.LATEST, os.rename(path, rpmfile) # Unpack and install - for rpmfile, dlsize, reponame in rpm_list: + for rpmfile, reponame in rpm_list: with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: package_hdr = get_package_hdr(rpmfile) package_name = package_hdr[rpm.RPMTAG_NAME] @@ -525,9 +524,10 @@ def install(args, app, version_selector=VersionSelector.LATEST, ' {reinstall,upgrade,downgrade}' ' operations.)') % name, file=sys.stderr) continue + # Check if local versus candidate version is in line with the # operation - elif override_existing: + if override_existing: if name not in app.domains: parser.error( "Template '%s' not already installed." % name) @@ -702,7 +702,7 @@ def search(args, app): # Get latest version for each template query_res_tmp = [] - for name, grp in itertools.groupby(sorted(query_res), lambda x: x[0]): + for _, grp in itertools.groupby(sorted(query_res), lambda x: x[0]): def compare(lhs, rhs): return lhs if rpm.labelCompare(lhs[1:4], rhs[1:4]) < 0 else rhs query_res_tmp.append(functools.reduce(compare, grp)) @@ -753,7 +753,7 @@ def search(args, app): search_res = sorted(search_res.items(), key=key_func) - def gen_header(idx, needles): + def gen_header(needles): fields = [] weight_types = set(x[0] for x in needles) for weight, field in WEIGHT_TO_FIELD: @@ -767,7 +767,7 @@ def search(args, app): last_header = '' for idx, needles in search_res: # Print headers - cur_header = gen_header(idx, needles) + cur_header = gen_header(needles) if last_header != cur_header: last_header = cur_header # XXX: The style is different from that of DNF From a9a19428f3edb6378262d8ff45210aec697224c5 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 1 Aug 2020 03:05:21 +0800 Subject: [PATCH 030/119] qvm-template: Check that template spec is not "---". --- qubesadmin/tools/qvm_template.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index b66bea0..1292b37 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -188,7 +188,8 @@ def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): def qrexec_payload(args, app, spec, refresh): _ = app # unused - # TODO: Check that spec != '---' + if spec == '---': + parser.error("Malformed template name: argument should not be '---'.") def check_newline(string, name): if '\n' in string: From 5319e7a41afa7bf04919b37f6de00005d23f9d5c Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 1 Aug 2020 03:06:04 +0800 Subject: [PATCH 031/119] qvm-template: Fix typo. --- qubesadmin/tools/qvm_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 1292b37..c4fe3e6 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -408,7 +408,7 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): or rpm.labelCompare(full_candid[name].evr, entry.evr) < 0: full_candid[name] = entry - return candid + return full_candid def download(args, app, path_override=None, dl_list=None, suffix='', version_selector=VersionSelector.LATEST): From 377e2a77ff2146b569c132b9d85aa5e45d3acb65 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 1 Aug 2020 03:21:31 +0800 Subject: [PATCH 032/119] qvm-template: Check that template is managed by qvm-template before accessing relevant features. --- qubesadmin/tools/qvm_template.py | 38 +++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index c4fe3e6..e06a1f8 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -172,6 +172,14 @@ def is_managed_template(vm): return 'template-name' in vm.features \ and vm.name == vm.features['template-name'] +def get_managed_template_vm(app, name): + if name not in app.domains: + parser.error("Template '%s' not already installed." % name) + vm = app.domains[name] + if not is_managed_template(vm): + parser.error("Template '%s' is not managed by qvm-template." % name) + return vm + def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): if args.updatevm: return app.domains[args.updatevm].run_service( @@ -351,7 +359,6 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): query_res = qrexec_repoquery(args, app, PACKAGE_NAME_PREFIX + template) # We only select one package for each distinct package name - # TODO: Check local VM is managed by qvm-template for entry in query_res: ver = (entry.epoch, entry.version, entry.release) insert = False @@ -360,19 +367,13 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): or rpm.labelCompare(candid[entry.name][0], ver) < 0: insert = True elif version_selector == VersionSelector.REINSTALL: - if entry.name not in app.domains: - parser.error( - "Template '%s' not already installed." % entry.name) - vm = app.domains[entry.name] + vm = get_managed_template_vm(app, entry.name) cur_ver = query_local_evr(vm) if rpm.labelCompare(ver, cur_ver) == 0: insert = True elif version_selector in [VersionSelector.LATEST_LOWER, VersionSelector.LATEST_HIGHER]: - if entry.name not in app.domains: - parser.error( - "Template '%s' not already installed." % entry.name) - vm = app.domains[entry.name] + vm = get_managed_template_vm(app, entry.name) cur_ver = query_local_evr(vm) cmp_res = -1 \ if version_selector == VersionSelector.LATEST_LOWER \ @@ -384,8 +385,8 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): if insert: candid[entry.name] = DlEntry(ver, entry.reponame, entry.dlsize) - # XXX: As it's possible to include version information in `template` - # Perhaps the messages can be improved + # XXX: As it's possible to include version information in `template`, + # perhaps the messages can be improved if len(candid) == 0: if version_selector == VersionSelector.LATEST: parser.error('Template \'%s\' not found.' % template) @@ -481,7 +482,8 @@ def install(args, app, version_selector=VersionSelector.LATEST, assert entry.reponame != '@commandline' if not override_existing and name in app.domains: print(('Template \'%s\' already installed, skipping...' - ' (You may want to use the {reinstall,upgrade,downgrade}' + ' (You may want to use the' + ' {reinstall,upgrade,downgrade}' ' operations.)') % name, file=sys.stderr) del dl_list_copy[name] else: @@ -529,10 +531,7 @@ def install(args, app, version_selector=VersionSelector.LATEST, # Check if local versus candidate version is in line with the # operation if override_existing: - if name not in app.domains: - parser.error( - "Template '%s' not already installed." % name) - vm = app.domains[name] + vm = get_managed_template_vm(app, name) pkg_evr = ( str(package_hdr[rpm.RPMTAG_EPOCHNUM]), package_hdr[rpm.RPMTAG_VERSION], @@ -657,7 +656,7 @@ def list_templates(args, app, operation): if is_managed_template(vm): if not args.templates or \ any(is_match_spec( - vm.features['template-name'], + vm.name, *query_local_evr(vm), spec)[0] for spec in args.templates): @@ -672,15 +671,14 @@ def list_templates(args, app, operation): for data in query_res: remote.add(data.name) for vm in app.domains: - if is_managed_template(vm) and \ - vm.features['template-name'] not in remote: + if is_managed_template(vm) and vm.name not in remote: append_vm(vm, TemplateState.EXTRA) if args.upgrades: local = {} for vm in app.domains: if is_managed_template(vm): - local[vm.features['template-name']] = query_local_evr(vm) + local[vm.name] = query_local_evr(vm) for entry in query_res: if entry.name in local: if rpm.labelCompare(local[entry.name], From bf0635218ac2364a41aa977eb8ab45fbb4374bc8 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 4 Aug 2020 01:34:14 +0800 Subject: [PATCH 033/119] qvm-template: Better args parsing: Use subparsers and complain about unknown args if the operation is not "remove". --- qubesadmin/tools/qvm_template.py | 152 +++++++++++++++++++++---------- 1 file changed, 105 insertions(+), 47 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index e06a1f8..044fd13 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -36,50 +36,99 @@ def qubes_release(): return subprocess.check_output(['lsb_release', '-sr'], encoding='UTF-8').strip() -parser = argparse.ArgumentParser(description='Qubes Template Manager') -parser.add_argument('operation', type=str) -parser.add_argument('templates', nargs='*') +def parser_gen(): + formatter = argparse.ArgumentDefaultsHelpFormatter + parser_main = argparse.ArgumentParser(description='Qubes Template Manager', + formatter_class=formatter) + subparsers = parser_main.add_subparsers(dest='operation', required=True, + description='Command to run.') -# qrexec related -parser.add_argument('--repo-files', action='append', - default=['/etc/yum.repos.d/qubes-templates.repo'], - help='Specify files containing DNF repository configuration.') -parser.add_argument('--updatevm', default='sys-firewall', - help='Specify VM to download updates from.') -# DNF-related options -parser.add_argument('--enablerepo', action='append', - help='Enable additional repositories.') -parser.add_argument('--disablerepo', action='append', - help='Disable certain repositories.') -parser.add_argument('--repoid', action='append', - help='Enable just specific repositories.') -parser.add_argument('--releasever', default=qubes_release(), - help='Override distro release version.') -parser.add_argument('--refresh', action='store_true', - help='Set repository metadata as expired before running the command.') -parser.add_argument('--cachedir', default=CACHE_DIR, - help='Override cache directory.') -# qvm-template install -parser.add_argument('--nogpgcheck', action='store_true', - help='Disable signature checks.') -parser.add_argument('--allow-pv', action='store_true', - help='Allow setting virt_mode to pv in configuration file.') -parser.add_argument('--pool', - help='Specify pool to store created VMs in.') -# qvm-template download -parser.add_argument('--downloaddir', default='.', - help='Override download directory.') -parser.add_argument('--retries', default=5, type=int, - help='Override number of retries for downloads.') -# qvm-template list -parser.add_argument('--all', action='store_true') -parser.add_argument('--installed', action='store_true') -parser.add_argument('--available', action='store_true') -parser.add_argument('--extras', action='store_true') -parser.add_argument('--upgrades', action='store_true') -# qvm-template search -# Already defined above -#parser.add_argument('--all', action='store_true') + def parser_add_command(cmd, help_str, add_help=True): + return subparsers.add_parser(cmd, formatter_class=formatter, + help=help_str, description=help_str, add_help=add_help) + + # qrexec/DNF related + parser_main.add_argument('--repo-files', action='append', + default=['/etc/yum.repos.d/qubes-templates.repo'], + help='Specify files containing DNF repository configuration.') + parser_main.add_argument('--updatevm', default='sys-firewall', + help='Specify VM to download updates from.') + parser_main.add_argument('--enablerepo', action='append', default=[], + help='Enable additional repositories.') + parser_main.add_argument('--disablerepo', action='append', default=[], + help='Disable certain repositories.') + parser_main.add_argument('--repoid', action='append', default=[], + help='Enable just specific repositories.') + parser_main.add_argument('--releasever', default=qubes_release(), + help='Override distro release version.') + parser_main.add_argument('--refresh', action='store_true', + help='Set repository metadata as expired before running the command.') + parser_main.add_argument('--cachedir', default=CACHE_DIR, + help='Specify cache directory.') + # qvm-template download + parser_download = parser_add_command('download', + help_str='Download template package.') + parser_download.add_argument('--downloaddir', default='.', + help='Specify download directory.') + parser_download.add_argument('--retries', default=5, type=int, + help='Specify number of retries for downloads.') + parser_download.add_argument('templates', nargs='*', metavar='TEMPLATE') + # qvm-template {install,reinstall,downgrade,upgrade} + parser_install = parser_add_command('install', + help_str='Install template packages.') + parser_install.add_argument('--pool', + help='Specify pool to store created VMs in.') + parser_reinstall = parser_add_command('reinstall', + help_str='Reinstall template packages.') + parser_downgrade = parser_add_command('downgrade', + help_str='Downgrade template packages.') + parser_upgrade = parser_add_command('upgrade', + help_str='Upgrade template packages.') + for parser_x in [parser_install, parser_reinstall, + parser_downgrade, parser_upgrade]: + parser_x.add_argument('--nogpgcheck', action='store_true', + help='Disable signature checks.') + parser_x.add_argument('--allow-pv', action='store_true', + help='Allow setting virt_mode to pv in configuration file.') + parser_x.add_argument('templates', nargs='*', metavar='TEMPLATE') + # qvm-template {list,info} + parser_list = parser_add_command('list', + help_str='List templates.') + parser_info = parser_add_command('info', + help_str='Display details about templates.') + for parser_x in [parser_list, parser_info]: + parser_x.add_argument('--all', action='store_true', + help='Show all templates (default).') + parser_x.add_argument('--installed', action='store_true', + help='Show installed templates.') + parser_x.add_argument('--available', action='store_true', + help='Show available templates.') + parser_x.add_argument('--extras', action='store_true', + help=('Show extras (e.g., ones that exists' + ' locally but not in repos) templates.')) + parser_x.add_argument('--upgrades', action='store_true', + help='Show upgradable templates.') + parser_x.add_argument('templates', nargs='*', metavar='TEMPLATE') + # qvm-template search + parser_search = parser_add_command('search', + help_str='Search template details for the given string.') + parser_search.add_argument('--all', action='store_true', + help=('Search also in template description and URL. In addition,' + ' the criterion are evaluated with OR instead of AND.')) + parser_search.add_argument('templates', nargs='*', metavar='PATTERN') + # qvm-template remove + parser_remove = parser_add_command('remove', + help_str='Remove installed templates.', + add_help=False) # Forward --help to qvm-remove + _ = parser_remove # unused + # qvm-template clean + parser_clean = parser_add_command('clean', + help_str='Remove cached data.') + _ = parser_clean # unused + + return parser_main + +parser = parser_gen() class TemplateState(enum.Enum): INSTALLED = 'installed' @@ -205,13 +254,13 @@ def qrexec_payload(args, app, spec, refresh): " argument should not contain '\\n'.") payload = '' - for repo in args.enablerepo if args.enablerepo else []: + for repo in args.enablerepo: check_newline(repo, '--enablerepo') payload += '--enablerepo=%s\n' % repo - for repo in args.disablerepo if args.disablerepo else []: + for repo in args.disablerepo: check_newline(repo, '--disablerepo') payload += '--disablerepo=%s\n' % repo - for repo in args.repoid if args.repoid else []: + for repo in args.repoid: check_newline(repo, '--repoid') payload += '--repoid=%s\n' % repo if refresh: @@ -791,7 +840,16 @@ def clean(args, app): shutil.rmtree(args.cachedir) def main(args=None, app=None): - args, _ = parser.parse_known_args(args) + raw_args = args + args, unk_args = parser.parse_known_args(raw_args) + if args.operation != 'remove' and unk_args: + args = parser.parse_args(raw_args) # this should result in an error + assert False and 'This line should not be executed.' + # FIXME: Currently doing things this way as we have to forward + # arguments to qvm-remove. While argparse.REMAINDER should be able to + # solve this, there's a bug (issue 17050) that prevents it from working + # on inputs where the first argument is an option, like 'qvm-template + # remove --help'. The bug should be fixed in Python 3.9. if app is None: app = qubesadmin.Qubes() From 582c87644dbd95d244761e8bd1282409578e5578 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 4 Aug 2020 01:35:14 +0800 Subject: [PATCH 034/119] qvm-template: Use repo file from qubes-repo-templates. --- qubesadmin/tools/qvm_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 044fd13..f59f75e 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -49,7 +49,7 @@ def parser_gen(): # qrexec/DNF related parser_main.add_argument('--repo-files', action='append', - default=['/etc/yum.repos.d/qubes-templates.repo'], + default=['/usr/share/qubes/repo-templates/qubes-templates.repo'], help='Specify files containing DNF repository configuration.') parser_main.add_argument('--updatevm', default='sys-firewall', help='Specify VM to download updates from.') From e482b9eb0f9f378373d23c08fe053ddd84dc6083 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 4 Aug 2020 01:38:52 +0800 Subject: [PATCH 035/119] qvm-template: Use "vm.features.get" instead of explicit membership check. --- qubesadmin/tools/qvm_template.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index f59f75e..da49188 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -218,8 +218,7 @@ def query_local_evr(vm): vm.features['template-release']) def is_managed_template(vm): - return 'template-name' in vm.features \ - and vm.name == vm.features['template-name'] + return vm.features.get('template-name', None) == vm.name def get_managed_template_vm(app, name): if name not in app.domains: From 69cd2858109f2af9b8daf32180b8ea26599ae22d Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 4 Aug 2020 01:40:59 +0800 Subject: [PATCH 036/119] qvm-template: Defer qrexec calls so that they can be omitted if exceptions are raised. --- qubesadmin/tools/qvm_template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index da49188..d3e8518 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -275,8 +275,8 @@ def qrexec_payload(args, app, spec, refresh): return payload def qrexec_repoquery(args, app, spec='*', refresh=False): - proc = qrexec_popen(args, app, 'qubes.TemplateSearch') payload = qrexec_payload(args, app, spec, refresh) + proc = qrexec_popen(args, app, 'qubes.TemplateSearch') stdout, stderr = proc.communicate(payload.encode('UTF-8')) stdout = stdout.decode('ASCII') if proc.wait() != 0: @@ -335,10 +335,10 @@ def qrexec_repoquery(args, app, spec='*', refresh=False): def qrexec_download(args, app, spec, path, dlsize=None, refresh=False): with open(path, 'wb') as fd: + payload = qrexec_payload(args, app, spec, refresh) # Don't filter ESCs for binary files proc = qrexec_popen(args, app, 'qubes.TemplateDownload', stdout=fd, filter_esc=False) - payload = qrexec_payload(args, app, spec, refresh) proc.stdin.write(payload.encode('UTF-8')) proc.stdin.close() with tqdm.tqdm(desc=spec, total=dlsize, unit_scale=True, From 41cf9f948e1c999c77815f044b00cc515799794f Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 4 Aug 2020 02:51:36 +0800 Subject: [PATCH 037/119] qvm-template: Partially include docstrings and type hints. --- qubesadmin/tools/qvm_template.py | 77 +++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 12 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index d3e8518..330aec4 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -14,6 +14,7 @@ import subprocess import sys import tempfile import time +import typing import qubesadmin import qubesadmin.tools @@ -28,7 +29,8 @@ CACHE_DIR = os.path.join(xdg.BaseDirectory.xdg_cache_home, 'qvm-template') UNVERIFIED_SUFFIX = '.unverified' LOCK_FILE = '/var/tmp/qvm-template.lck' -def qubes_release(): +def qubes_release() -> str: + """Return the Qubes release.""" if os.path.exists('/usr/share/qubes/marker-vm'): with open('/usr/share/qubes/marker-vm', 'r') as fd: # Get last line (in the format `x.x`) @@ -36,7 +38,8 @@ def qubes_release(): return subprocess.check_output(['lsb_release', '-sr'], encoding='UTF-8').strip() -def parser_gen(): +def parser_gen() -> argparse.ArgumentParser: + """Generate argument parser for the application.""" formatter = argparse.ArgumentDefaultsHelpFormatter parser_main = argparse.ArgumentParser(description='Qubes Template Manager', formatter_class=formatter) @@ -131,12 +134,14 @@ def parser_gen(): parser = parser_gen() class TemplateState(enum.Enum): + """Enum representing the state of a template.""" INSTALLED = 'installed' AVAILABLE = 'available' EXTRA = 'extra' UPGRADABLE = 'upgradable' - def title(self): + def title(self) -> str: + """Return a long description of the state. Can be used as headings.""" #pylint: disable=invalid-name TEMPLATE_TITLES = { TemplateState.INSTALLED: 'Installed Templates', @@ -147,11 +152,13 @@ class TemplateState(enum.Enum): return TEMPLATE_TITLES[self] class VersionSelector(enum.Enum): + """Enum representing how the candidate template version is chosen.""" LATEST = enum.auto() REINSTALL = enum.auto() LATEST_LOWER = enum.auto() LATEST_HIGHER = enum.auto() +# TODO: Docstrings and type hints for Template and DlEntry Template = collections.namedtuple('Template', [ 'name', 'epoch', @@ -172,12 +179,24 @@ DlEntry = collections.namedtuple('DlEntry', [ 'dlsize' ]) -def build_version_str(evr): +def build_version_str(evr: typing.Tuple[str, str, str]) -> str: + """Return version string described by ``evr`` in (epoch, version, release) + format.""" return '%s:%s-%s' % evr -def is_match_spec(name, epoch, version, release, spec): - # Refer to "NEVRA Matching" in the DNF documentation - # NOTE: Currently "arch" is ignored as the templates should be of "noarch" +def is_match_spec(name: str, epoch: str, version: str, release: str, spec: str + ) -> typing.Tuple[bool, float]: + """Check whether (name, epoch, version, release) matches the spec string. + + For the algorithm, refer to section "NEVRA Matching" in the DNF + documentation. + + NOTE: Currently ``arch`` is ignored as the templates should be of + ``noarch``. + + :return: the first element indicates whether there is a match; the second + element represents the priority of the match (lower is better). + """ if epoch != 0: targets = [ f'{name}-{epoch}:{version}-{release}', @@ -197,7 +216,11 @@ def is_match_spec(name, epoch, version, release, spec): return True, prio return False, float('inf') -def query_local(vm): +def query_local(vm: qubesadmin.vm.QubesVM) -> Template: + """Return Template object associated with ``vm``. + + Requires the VM to be managed by qvm-template. + """ return Template( vm.features['template-name'], vm.features['template-epoch'], @@ -211,16 +234,24 @@ def query_local(vm): vm.features['template-summary'], vm.features['template-description'].replace('|', '\n')) -def query_local_evr(vm): +def query_local_evr(vm: qubesadmin.vm.QubesVM) -> typing.Tuple[str, str, str]: + """Return the (epoch, version, release) of ``vm``. + + Requires the VM to be managed by qvm-template. + """ return ( vm.features['template-epoch'], vm.features['template-version'], vm.features['template-release']) -def is_managed_template(vm): +def is_managed_template(vm: qubesadmin.vm.QubesVM) -> bool: + """Return whether the VM is managed by qvm-template.""" return vm.features.get('template-name', None) == vm.name -def get_managed_template_vm(app, name): +def get_managed_template_vm(app: qubesadmin.app.QubesBase, name: str + ) -> qubesadmin.vm.QubesVM: + """Return the QubesVM object associated with the given name if it exists + and is managed by qvm-template, otherwise raise a parser error.""" if name not in app.domains: parser.error("Template '%s' not already installed." % name) vm = app.domains[name] @@ -228,7 +259,29 @@ def get_managed_template_vm(app, name): parser.error("Template '%s' is not managed by qvm-template." % name) return vm -def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True): +def qrexec_popen( + args: argparse.Namespace, + app: qubesadmin.app.QubesBase, + service: str, + stdout: int = subprocess.PIPE, + filter_esc: bool = True) -> subprocess.Popen: + """Return Popen object that communicates with the given qrexec call. + + Note that this falls back to invoking /etc/qubes-rpc/* directly if + args.updatevm is None. + + :param args: arguments received by the application + :param app: Qubes application object + :param service: the qrexec call to invoke + :param stdout: Where the process stdout points to. This is passed directly + to subprocess.Popen. Defaults to subprocess.PIPE. + + Note that stderr is always set to subprocess.PIPE. + :param filter_esc: whether to filter out escape sequences from + stdout/stderr. Defaults to True. + + :returns: Popen object that communicates with the given qrexec call + """ if args.updatevm: return app.domains[args.updatevm].run_service( service, From 7b6fa39d1c2568fcb482ba1f9d24201053036551 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Thu, 6 Aug 2020 02:05:57 +0800 Subject: [PATCH 038/119] qvm-template: More docstrings. --- qubesadmin/tools/qvm_template.py | 373 +++++++++++++++++++++++-------- 1 file changed, 274 insertions(+), 99 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 330aec4..4112385 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +'''Tool for managing VM templates.''' + import argparse import collections import datetime @@ -154,34 +156,37 @@ class TemplateState(enum.Enum): class VersionSelector(enum.Enum): """Enum representing how the candidate template version is chosen.""" LATEST = enum.auto() + """Install latest version.""" REINSTALL = enum.auto() + """Reinstall current version.""" LATEST_LOWER = enum.auto() + """Downgrade to the highest version that is lower than the current one.""" LATEST_HIGHER = enum.auto() + """Upgrade to the highest version that is higher than the current one.""" -# TODO: Docstrings and type hints for Template and DlEntry -Template = collections.namedtuple('Template', [ - 'name', - 'epoch', - 'version', - 'release', - 'reponame', - 'dlsize', - 'buildtime', - 'licence', - 'url', - 'summary', - 'description' -]) +class Template(typing.NamedTuple): + """Details of a template.""" + name: str + epoch: str + version: str + release: str + reponame: str + dlsize: int + buildtime: datetime.datetime + licence: str + url: str + summary: str + description: str -DlEntry = collections.namedtuple('DlEntry', [ - 'evr', - 'reponame', - 'dlsize' -]) +class DlEntry(typing.NamedTuple): + """Information about a template to be downloaded.""" + evr: typing.Tuple[str, str, str] + reponame: str + dlsize: int def build_version_str(evr: typing.Tuple[str, str, str]) -> str: - """Return version string described by ``evr`` in (epoch, version, release) - format.""" + """Return version string described by ``evr``, which is in (epoch, version, + release) format.""" return '%s:%s-%s' % evr def is_match_spec(name: str, epoch: str, version: str, release: str, spec: str @@ -191,11 +196,11 @@ def is_match_spec(name: str, epoch: str, version: str, release: str, spec: str For the algorithm, refer to section "NEVRA Matching" in the DNF documentation. - NOTE: Currently ``arch`` is ignored as the templates should be of + Note that currently ``arch`` is ignored as the templates should be of ``noarch``. - :return: the first element indicates whether there is a match; the second - element represents the priority of the match (lower is better). + :return: A tuple. The first element indicates whether there is a match; the + second element represents the priority of the match (lower is better) """ if epoch != 0: targets = [ @@ -263,24 +268,26 @@ def qrexec_popen( args: argparse.Namespace, app: qubesadmin.app.QubesBase, service: str, - stdout: int = subprocess.PIPE, + stdout: typing.Union[int, typing.IO] = subprocess.PIPE, filter_esc: bool = True) -> subprocess.Popen: - """Return Popen object that communicates with the given qrexec call. + """Return ``Popen`` object that communicates with the given qrexec call in + ``args.updatevm``. - Note that this falls back to invoking /etc/qubes-rpc/* directly if - args.updatevm is None. + Note that this falls back to invoking ``/etc/qubes-rpc/*`` directly if + ``args.updatevm`` is None. - :param args: arguments received by the application + :param args: Arguments received by the application. ``args.updatevm`` is + used :param app: Qubes application object - :param service: the qrexec call to invoke + :param service: The qrexec call to invoke :param stdout: Where the process stdout points to. This is passed directly - to subprocess.Popen. Defaults to subprocess.PIPE. + to ``subprocess.Popen``. Defaults to ``subprocess.PIPE`` - Note that stderr is always set to subprocess.PIPE. - :param filter_esc: whether to filter out escape sequences from - stdout/stderr. Defaults to True. + Note that stderr is always set to ``subprocess.PIPE`` + :param filter_esc: Whether to filter out escape sequences from + stdout/stderr. Defaults to True - :returns: Popen object that communicates with the given qrexec call + :returns: ``Popen`` object that communicates with the given qrexec call """ if args.updatevm: return app.domains[args.updatevm].run_service( @@ -294,7 +301,21 @@ def qrexec_popen( stdout=stdout, stderr=subprocess.PIPE) -def qrexec_payload(args, app, spec, refresh): +def qrexec_payload(args: argparse.Namespace, app: qubesadmin.app.QubesBase, + spec: str, refresh: bool) -> str: + """Return payload string for the ``qubes.Template*`` qrexec calls. + + :param args: Arguments received by the application. Specifically, + ``args.{enablerepo,disablerepo,repoid,releasever,repo_files}`` are used + :param app: Qubes application object + :param spec: Package spec to query (refer to ```` in the + DNF documentation) + :param refresh: Whether to force refresh repo metadata + + :return: Payload string + + :raises: Parser error if spec equals ``---`` or input contains ``\\n`` + """ _ = app # unused if spec == '---': @@ -327,7 +348,25 @@ def qrexec_payload(args, app, spec, refresh): payload += fd.read() + '\n' return payload -def qrexec_repoquery(args, app, spec='*', refresh=False): +def qrexec_repoquery( + args: argparse.Namespace, + app: qubesadmin.app.QubesBase, + spec: str = '*', + refresh: bool = False) -> typing.List[Template]: + """Query template information from repositories. + + :param args: Arguments received by the application. Specifically, + ``args.{enablerepo,disablerepo,repoid,releasever,repo_files,updatevm}`` + are used + :param app: Qubes application object + :param spec: Package spec to query (refer to ```` in the + DNF documentation). Defaults to ``*`` + :param refresh: Whether to force refresh repo metadata. Defaults to False + + :raises ConnectionError: if the qrexec call fails + + :return: List of ``Template`` objects representing the result of the query + """ payload = qrexec_payload(args, app, spec, refresh) proc = qrexec_popen(args, app, 'qubes.TemplateSearch') stdout, stderr = proc.communicate(payload.encode('UTF-8')) @@ -386,7 +425,28 @@ def qrexec_repoquery(args, app, spec='*', refresh=False): " unexpected data format.")) return result -def qrexec_download(args, app, spec, path, dlsize=None, refresh=False): +def qrexec_download( + args: argparse.Namespace, + app: qubesadmin.app.QubesBase, + spec: str, + path: str, + dlsize: typing.Optional[int] = None, + refresh: bool = False) -> None: + """Download a template from repositories. + + :param args: Arguments received by the application. Specifically, + ``args.{enablerepo,disablerepo,repoid,releasever,repo_files,updatevm}`` + are used + :param app: Qubes application object + :param spec: Package spec to query (refer to ```` in the + DNF documentation) + :param path: Path to place the downloaded template + :param dlsize: Size of template to be downloaded. Used for the progress + bar. Optional + :param refresh: Whether to force refresh repo metadata. Defaults to False + + :raises ConnectionError: if the qrexec call fails + """ with open(path, 'wb') as fd: payload = qrexec_payload(args, app, spec, refresh) # Don't filter ESCs for binary files @@ -405,12 +465,25 @@ def qrexec_download(args, app, spec, path, dlsize=None, refresh=False): if proc.wait() != 0: raise ConnectionError( "qrexec call 'qubes.TemplateDownload' failed.") - return True -def verify_rpm(path, nogpgcheck=False, transaction_set=None): - # NOTE: Verifying RPMs this way is prone to TOCTOU. This is okay for local - # files, but may create problems if multiple instances of `qvm-template` - # are downloading the same file, so a lock is needed in that case. +def verify_rpm( + path: str, + nogpgcheck: bool = False, + transaction_set: typing.Optional[rpm.transaction.TransactionSet] = None + ) -> bool: + """Verify the digest and signature of a RPM package. + + Note that verifying RPMs this way is prone to TOCTOU. This is okay for + local files, but may create problems if multiple instances of + **qvm-template** are downloading the same file, so a lock is needed in that + case. + + :param path: Location of the RPM package + :param nogpgcheck: Whether to allow invalid GPG signatures + :param transaction_set: Override RPM ``TransactionSet``. Optional + + :return: Whether the RPM is verified + """ if transaction_set is None: transaction_set = rpm.TransactionSet() with open(path, 'rb') as fd: @@ -427,14 +500,34 @@ def verify_rpm(path, nogpgcheck=False, transaction_set=None): return False return True -def get_package_hdr(path, transaction_set=None): +def get_package_hdr( + path: str, + transaction_set: typing.Optional[rpm.transaction.TransactionSet] = None + ) -> rpm.hdr: + """Return header of a RPM package. + + Note that this function **does not** check the integrity of the package. + + :param path: Location of the RPM package + :param transaction_set: Override RPM ``TransactionSet``. Optional + + :return: RPM headers + """ if transaction_set is None: transaction_set = rpm.TransactionSet() with open(path, 'rb') as fd: hdr = transaction_set.hdrFromFdno(fd) return hdr -def extract_rpm(name, path, target): +def extract_rpm(name: str, path: str, target: str) -> bool: + """Extract a template RPM package. + + :param name: Name of the template + :param path: Location of the RPM package + :param target: Target path to extract to + + :return: Whether the extraction succeeded + """ rpm2cpio = subprocess.Popen(['rpm2cpio', path], stdout=subprocess.PIPE) # `-D` is GNUism cpio = subprocess.Popen([ @@ -446,12 +539,26 @@ def extract_rpm(name, path, target): ], stdin=rpm2cpio.stdout, stdout=subprocess.DEVNULL) return rpm2cpio.wait() == 0 and cpio.wait() == 0 -def get_dl_list(args, app, version_selector=VersionSelector.LATEST): - full_candid = {} +def get_dl_list( + args: argparse.Namespace, + app: qubesadmin.app.QubesBase, + version_selector: VersionSelector = VersionSelector.LATEST + ) -> typing.Dict[str, DlEntry]: + """Return list of templates that needs to be downloaded. + + :param args: Arguments received by the application. + :param app: Qubes application object + :param version_selector: Specify algorithm to select the candidate version + of a package. Defaults to ``VersionSelector.LATEST`` + + :return: Dictionary that maps to ``DlEntry`` the names of templates that + needs to be downloaded + """ + full_candid: typing.Dict[str, DlEntry] = {} for template in args.templates: # This will be merged into `full_candid` later. # It is separated so that we can check whether it is empty. - candid = {} + candid: typing.Dict[str, DlEntry] = {} # Skip local RPMs if template.endswith('.rpm'): @@ -505,15 +612,35 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST): file=sys.stderr) # Merge & choose the template with the highest version - for name, entry in candid.items(): + for name, dlentry in candid.items(): if name not in full_candid \ - or rpm.labelCompare(full_candid[name].evr, entry.evr) < 0: - full_candid[name] = entry + or rpm.labelCompare(full_candid[name].evr, dlentry.evr) < 0: + full_candid[name] = dlentry return full_candid -def download(args, app, path_override=None, - dl_list=None, suffix='', version_selector=VersionSelector.LATEST): +def download( + args: argparse.Namespace, + app: qubesadmin.app.QubesBase, + path_override: typing.Optional[str] = None, + dl_list: typing.Optional[typing.Dict[str, DlEntry]] = None, + suffix: str = '', + version_selector: VersionSelector = VersionSelector.LATEST) -> None: + """Command that downloads template packages. + + :param args: Arguments received by the application. + :param app: Qubes application object + :param path_override: Override path to store downloads. If not set or set + to None, ``args.downloaddir`` is used. Optional + :param dl_list: Override list of templates to download. If not set or set + to None, ``get_dl_list`` is called, which generates the list from + ``args``. Optional + :param suffix: Suffix to add to the file name of downloaded packages. This + is useful if you want to distinguish between verified and unverified + packages. Defaults to an empty string + :param version_selector: Specify algorithm to select the candidate version + of a package. Defaults to ``VersionSelector.LATEST`` + """ if dl_list is None: dl_list = get_dl_list(args, app, version_selector=version_selector) @@ -553,8 +680,23 @@ def download(args, app, path_override=None, print('\'%s\' download failed.' % spec, file=sys.stderr) sys.exit(1) -def install(args, app, version_selector=VersionSelector.LATEST, - override_existing=False): +def install( + args: argparse.Namespace, + app: qubesadmin.app.QubesBase, + version_selector: VersionSelector = VersionSelector.LATEST, + override_existing: bool = False) -> None: + """Command that installs template packages. + + This command creates a lock file to ensure that two instances are not + running at the same time. + + :param args: Arguments received by the application. + :param app: Qubes application object + :param version_selector: Specify algorithm to select the candidate version + of a package. Defaults to ``VersionSelector.LATEST`` + :param override_existing: Whether to override existing packages. Used for + reinstall, upgrade, and downgrade operations + """ try: with open(LOCK_FILE, 'x') as _: pass @@ -700,7 +842,16 @@ def install(args, app, version_selector=VersionSelector.LATEST, finally: os.remove(LOCK_FILE) -def list_templates(args, app, operation): +def list_templates(args: argparse.Namespace, + app: qubesadmin.app.QubesBase, operation: str) -> None: + """Command that lists templates. + + :param args: Arguments received by the application. + :param app: Qubes application object + :param operation: If set to ``list``, display a listing similar to ``dnf + list``. If set to ``info``, display detailed template information + similar to ``dnf info``. Otherwise, an ``AssertionError`` is raised. + """ tpl_list = [] def append_list(data, status, install_time=None): @@ -745,10 +896,10 @@ def list_templates(args, app, operation): if args.all or args.available or args.extras or args.upgrades: if args.templates: - query_res = set() + query_res_set: typing.Set[Template] = set() for spec in args.templates: - query_res |= set(qrexec_repoquery(args, app, spec)) - query_res = list(query_res) + query_res_set |= set(qrexec_repoquery(args, app, spec)) + query_res = list(query_res_set) else: query_res = qrexec_repoquery(args, app) @@ -793,7 +944,12 @@ def list_templates(args, app, operation): print(k.title()) qubesadmin.tools.print_table(list(map(lambda x: x[1:], grp))) -def search(args, app): +def search(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: + """Command that searches template details for given patterns. + + :param args: Arguments received by the application. + :param app: Qubes application object + """ # Search in both installed and available templates query_res = qrexec_repoquery(args, app) for vm in app.domains: @@ -822,27 +978,29 @@ def search(args, app): (WEIGHT_DESCRIPTION, 'Description'), (WEIGHT_URL, 'URL')] - search_res = collections.defaultdict(list) + search_res_by_idx: \ + typing.Dict[int, typing.List[typing.Tuple[int, str, bool]]] = \ + collections.defaultdict(list) for keyword in args.templates: for idx, entry in enumerate(query_res): - needles = \ + needle_types = \ [(entry.name, WEIGHT_NAME), (entry.summary, WEIGHT_SUMMARY)] if args.all: - needles += [(entry.description, WEIGHT_DESCRIPTION), + needle_types += [(entry.description, WEIGHT_DESCRIPTION), (entry.url, WEIGHT_URL)] - for key, weight in needles: + for key, weight in needle_types: if fnmatch.fnmatch(key, '*' + keyword + '*'): exact = keyword == key if exact and weight == WEIGHT_NAME: weight = WEIGHT_NAME_EXACT - search_res[idx].append((weight, keyword, exact)) + search_res_by_idx[idx].append((weight, keyword, exact)) if not args.all: keywords = set(args.templates) - idxs = list(search_res.keys()) + idxs = list(search_res_by_idx.keys()) for idx in idxs: - if keywords != set(x[1] for x in search_res[idx]): - del search_res[idx] + if keywords != set(x[1] for x in search_res_by_idx[idx]): + del search_res_by_idx[idx] def key_func(x): # ORDER BY weight DESC, list_of_needles ASC, name ASC @@ -851,7 +1009,7 @@ def search(args, app): name = query_res[idx][0] return (-weight, needles, name) - search_res = sorted(search_res.items(), key=key_func) + search_res = sorted(search_res_by_idx.items(), key=key_func) def gen_header(needles): fields = [] @@ -874,7 +1032,12 @@ def search(args, app): print('===', cur_header, '===') print(query_res[idx].name, ':', query_res[idx].summary) -def remove(args, app): +def remove(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: + """Command that remove templates. + + :param args: Arguments received by the application. + :param app: Qubes application object + """ _ = args, app # unused # Remove 'remove' entry from the args... @@ -885,17 +1048,29 @@ def remove(args, app): # Use exec so stdio can be shared easily os.execvp('qvm-remove', ['qvm-remove'] + argv) -def clean(args, app): +def clean(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: + """Command that cleans the local package cache. + + :param args: Arguments received by the application. + :param app: Qubes application object + """ # TODO: More fine-grained options _ = app # unused shutil.rmtree(args.cachedir) -def main(args=None, app=None): - raw_args = args - args, unk_args = parser.parse_known_args(raw_args) - if args.operation != 'remove' and unk_args: - args = parser.parse_args(raw_args) # this should result in an error +def main(args: typing.Optional[typing.Sequence[str]] = None, + app: typing.Optional[qubesadmin.app.QubesBase] = None) -> int: + """Main routine of **qvm-template**. + + :param args: Override arguments received by the application. Optional + :param app: Override Qubes application object. Optional + + :return: Return code of the application + """ + p_args, unk_args = parser.parse_known_args(args) + if p_args.operation != 'remove' and unk_args: + p_args = parser.parse_args(args) # this should result in an error assert False and 'This line should not be executed.' # FIXME: Currently doing things this way as we have to forward # arguments to qvm-remove. While argparse.REMAINDER should be able to @@ -906,34 +1081,34 @@ def main(args=None, app=None): if app is None: app = qubesadmin.Qubes() - if args.refresh: - qrexec_repoquery(args, app, refresh=True) + if p_args.refresh: + qrexec_repoquery(p_args, app, refresh=True) - if args.operation == 'download': - download(args, app) - elif args.operation == 'install': - install(args, app) - elif args.operation == 'reinstall': - install(args, app, version_selector=VersionSelector.REINSTALL, + if p_args.operation == 'download': + download(p_args, app) + elif p_args.operation == 'install': + install(p_args, app) + elif p_args.operation == 'reinstall': + install(p_args, app, version_selector=VersionSelector.REINSTALL, override_existing=True) - elif args.operation == 'downgrade': - install(args, app, version_selector=VersionSelector.LATEST_LOWER, + elif p_args.operation == 'downgrade': + install(p_args, app, version_selector=VersionSelector.LATEST_LOWER, override_existing=True) - elif args.operation == 'upgrade': - install(args, app, version_selector=VersionSelector.LATEST_HIGHER, + elif p_args.operation == 'upgrade': + install(p_args, app, version_selector=VersionSelector.LATEST_HIGHER, override_existing=True) - elif args.operation == 'list': - list_templates(args, app, 'list') - elif args.operation == 'info': - list_templates(args, app, 'info') - elif args.operation == 'search': - search(args, app) - elif args.operation == 'remove': - remove(args, app) - elif args.operation == 'clean': - clean(args, app) + elif p_args.operation == 'list': + list_templates(p_args, app, 'list') + elif p_args.operation == 'info': + list_templates(p_args, app, 'info') + elif p_args.operation == 'search': + search(p_args, app) + elif p_args.operation == 'remove': + remove(p_args, app) + elif p_args.operation == 'clean': + clean(p_args, app) else: - parser.error('Operation \'%s\' not supported.' % args.operation) + parser.error('Operation \'%s\' not supported.' % p_args.operation) return 0 From 336b5c68c11bb1916cb9d71a5c127a6011bef0ae Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Thu, 6 Aug 2020 02:42:05 +0800 Subject: [PATCH 039/119] qvm-template: Initial support for machine-readable listings. --- qubesadmin/tools/qvm_template.py | 77 +++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 22 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 4112385..74084a5 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -113,6 +113,8 @@ def parser_gen() -> argparse.ArgumentParser: ' locally but not in repos) templates.')) parser_x.add_argument('--upgrades', action='store_true', help='Show upgradable templates.') + parser_x.add_argument('--machine-readable', action='store_true', + help='Enable machine-readable output.') parser_x.add_argument('templates', nargs='*', metavar='TEMPLATE') # qvm-template search parser_search = parser_add_command('search', @@ -861,25 +863,46 @@ def list_templates(args: argparse.Namespace, tpl_list.append((status, data.name, version_str, data.reponame)) def append_info(data, status, install_time=None): - tpl_list.append((status, 'Name', ':', data.name)) - tpl_list.append((status, 'Epoch', ':', data.epoch)) - tpl_list.append((status, 'Version', ':', data.version)) - tpl_list.append((status, 'Release', ':', data.release)) - tpl_list.append((status, 'Size', ':', - qubesadmin.utils.size_to_human(data.dlsize))) - tpl_list.append((status, 'Repository', ':', data.reponame)) - tpl_list.append((status, 'Buildtime', ':', str(data.buildtime))) - if install_time: - tpl_list.append((status, 'Install time', ':', str(install_time))) - tpl_list.append((status, 'URL', ':', data.url)) - tpl_list.append((status, 'License', ':', data.licence)) - tpl_list.append((status, 'Summary', ':', data.summary)) - # Only show "Description" for the first line - title = 'Description' - for line in data.description.splitlines(): - tpl_list.append((status, title, ':', line)) - title = '' - tpl_list.append((status, ' ', ' ', ' ')) # empty line + tpl_list.append((status, data, install_time)) + + def info_to_human_output(tpls): + output = [] + # TODO: Do groupby here to avoid including ``status`` in each row. + for status, data, install_time in tpls: + output.append((status, 'Name', ':', data.name)) + output.append((status, 'Epoch', ':', data.epoch)) + output.append((status, 'Version', ':', data.version)) + output.append((status, 'Release', ':', data.release)) + output.append((status, 'Size', ':', + qubesadmin.utils.size_to_human(data.dlsize))) + output.append((status, 'Repository', ':', data.reponame)) + output.append((status, 'Buildtime', ':', str(data.buildtime))) + if install_time: + output.append((status, 'Install time', ':', str(install_time))) + output.append((status, 'URL', ':', data.url)) + output.append((status, 'License', ':', data.licence)) + output.append((status, 'Summary', ':', data.summary)) + # Only show "Description" for the first line + title = 'Description' + for line in data.description.splitlines(): + output.append((status, title, ':', line)) + title = '' + output.append((status, ' ', ' ', ' ')) # empty line + return output + + def info_to_machine_output(tpls): + output = [] + for status, data, install_time in tpls: + name, epoch, version, release, reponame, dlsize, \ + buildtime, licence, url, summary, description = data + dlsize = str(dlsize) + buildtime = str(buildtime) + install_time = str(install_time) if install_time else '' + # TODO: Escape newlines in description? + output.append((status, name, epoch, version, release, reponame, + dlsize, buildtime, install_time, licence, url, summary, + description)) + return output if operation == 'list': append = append_list @@ -940,9 +963,19 @@ def list_templates(args: argparse.Namespace, if len(tpl_list) == 0: parser.error('No matching templates to list') - for k, grp in itertools.groupby(tpl_list, lambda x: x[0]): - print(k.title()) - qubesadmin.tools.print_table(list(map(lambda x: x[1:], grp))) + if not args.machine_readable: + if operation == 'info': + tpl_list = info_to_human_output(tpl_list) + for status, grp in itertools.groupby(tpl_list, lambda x: x[0]): + print(status.title()) + qubesadmin.tools.print_table(list(map(lambda x: x[1:], grp))) + else: + if operation == 'info': + tpl_list = info_to_machine_output(tpl_list) + for status, grp in itertools.groupby(tpl_list, lambda x: x[0]): + print('|' + status.value) + for line in grp: + print('|'.join(line[1:]) + '|') def search(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: """Command that searches template details for given patterns. From d11e74ae99f4e1ec097a5485fc141bd2cb8a7c21 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Thu, 6 Aug 2020 02:44:12 +0800 Subject: [PATCH 040/119] qvm-template: Include module in documentation. --- doc/qubesadmin.tools.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/qubesadmin.tools.rst b/doc/qubesadmin.tools.rst index 078e7f0..8c4250b 100644 --- a/doc/qubesadmin.tools.rst +++ b/doc/qubesadmin.tools.rst @@ -180,6 +180,14 @@ qubesadmin\.tools\.qvm\_tags module :undoc-members: :show-inheritance: +qubesadmin\.tools\.qvm\_template module +--------------------------------------- + +.. automodule:: qubesadmin.tools.qvm_template + :members: + :undoc-members: + :show-inheritance: + qubesadmin\.tools\.qvm\_template\_postprocess module ---------------------------------------------------- From ed35802ca22ae644af1ff78dc054f4b98c566bf1 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Fri, 7 Aug 2020 02:10:57 +0800 Subject: [PATCH 041/119] qvm-template: Tidy up code responsible for output in {info,list} operations. --- qubesadmin/tools/qvm_template.py | 93 ++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 39 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 74084a5..0c7f5e3 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -865,44 +865,55 @@ def list_templates(args: argparse.Namespace, def append_info(data, status, install_time=None): tpl_list.append((status, data, install_time)) + def list_to_output(tpls): + outputs = [] + for status, grp in itertools.groupby(tpls, lambda x: x[0]): + outputs.append((status, list(map(lambda x: x[1:], grp)))) + return outputs + def info_to_human_output(tpls): - output = [] - # TODO: Do groupby here to avoid including ``status`` in each row. - for status, data, install_time in tpls: - output.append((status, 'Name', ':', data.name)) - output.append((status, 'Epoch', ':', data.epoch)) - output.append((status, 'Version', ':', data.version)) - output.append((status, 'Release', ':', data.release)) - output.append((status, 'Size', ':', - qubesadmin.utils.size_to_human(data.dlsize))) - output.append((status, 'Repository', ':', data.reponame)) - output.append((status, 'Buildtime', ':', str(data.buildtime))) - if install_time: - output.append((status, 'Install time', ':', str(install_time))) - output.append((status, 'URL', ':', data.url)) - output.append((status, 'License', ':', data.licence)) - output.append((status, 'Summary', ':', data.summary)) - # Only show "Description" for the first line - title = 'Description' - for line in data.description.splitlines(): - output.append((status, title, ':', line)) - title = '' - output.append((status, ' ', ' ', ' ')) # empty line - return output + outputs = [] + for status, grp in itertools.groupby(tpls, lambda x: x[0]): + output = [] + for _, data, install_time in grp: + output.append(('Name', ':', data.name)) + output.append(('Epoch', ':', data.epoch)) + output.append(('Version', ':', data.version)) + output.append(('Release', ':', data.release)) + output.append(('Size', ':', + qubesadmin.utils.size_to_human(data.dlsize))) + output.append(('Repository', ':', data.reponame)) + output.append(('Buildtime', ':', str(data.buildtime))) + if install_time: + output.append(('Install time', ':', str(install_time))) + output.append(('URL', ':', data.url)) + output.append(('License', ':', data.licence)) + output.append(('Summary', ':', data.summary)) + # Only show "Description" for the first line + title = 'Description' + for line in data.description.splitlines(): + output.append((title, ':', line)) + title = '' + output.append((' ', ' ', ' ')) # empty line + outputs.append((status, output)) + return outputs def info_to_machine_output(tpls): - output = [] - for status, data, install_time in tpls: - name, epoch, version, release, reponame, dlsize, \ - buildtime, licence, url, summary, description = data - dlsize = str(dlsize) - buildtime = str(buildtime) - install_time = str(install_time) if install_time else '' - # TODO: Escape newlines in description? - output.append((status, name, epoch, version, release, reponame, - dlsize, buildtime, install_time, licence, url, summary, - description)) - return output + outputs = [] + for status, grp in itertools.groupby(tpls, lambda x: x[0]): + output = [] + for _, data, install_time in grp: + name, epoch, version, release, reponame, dlsize, \ + buildtime, licence, url, summary, description = data + dlsize = str(dlsize) + buildtime = str(buildtime) + install_time = str(install_time) if install_time else '' + # TODO: Escape newlines in description? + output.append((name, epoch, version, release, reponame, + dlsize, buildtime, install_time, licence, url, summary, + description)) + outputs.append((status, output)) + return outputs if operation == 'list': append = append_list @@ -966,16 +977,20 @@ def list_templates(args: argparse.Namespace, if not args.machine_readable: if operation == 'info': tpl_list = info_to_human_output(tpl_list) - for status, grp in itertools.groupby(tpl_list, lambda x: x[0]): + elif operation == 'list': + tpl_list = list_to_output(tpl_list) + for status, grp in tpl_list: print(status.title()) - qubesadmin.tools.print_table(list(map(lambda x: x[1:], grp))) + qubesadmin.tools.print_table(grp) else: if operation == 'info': tpl_list = info_to_machine_output(tpl_list) - for status, grp in itertools.groupby(tpl_list, lambda x: x[0]): + elif operation == 'list': + tpl_list = list_to_output(tpl_list) + for status, grp in tpl_list: print('|' + status.value) for line in grp: - print('|'.join(line[1:]) + '|') + print('|'.join(line) + '|') def search(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: """Command that searches template details for given patterns. From ba7b113206763d19c025ca64ed34c749a94fb7ad Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Fri, 7 Aug 2020 02:14:37 +0800 Subject: [PATCH 042/119] qvm-template: Replace newlines in machine-readable output. --- qubesadmin/tools/qvm_template.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 0c7f5e3..9f443a1 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -898,7 +898,7 @@ def list_templates(args: argparse.Namespace, outputs.append((status, output)) return outputs - def info_to_machine_output(tpls): + def info_to_machine_output(tpls, replace_newline=True): outputs = [] for status, grp in itertools.groupby(tpls, lambda x: x[0]): output = [] @@ -908,7 +908,8 @@ def list_templates(args: argparse.Namespace, dlsize = str(dlsize) buildtime = str(buildtime) install_time = str(install_time) if install_time else '' - # TODO: Escape newlines in description? + if replace_newline: + description = description.replace('\n', '|') output.append((name, epoch, version, release, reponame, dlsize, buildtime, install_time, licence, url, summary, description)) From c523d78d59395553062595922cd8ce1179eaa63f Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Fri, 7 Aug 2020 14:48:08 +0800 Subject: [PATCH 043/119] qvm-template: Initial implementation of repolist. --- qubesadmin/tools/qvm_template.py | 75 +++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 9f443a1..39f360a 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -9,6 +9,7 @@ import enum import fnmatch import functools import itertools +import operator import os import re import shutil @@ -132,6 +133,17 @@ def parser_gen() -> argparse.ArgumentParser: parser_clean = parser_add_command('clean', help_str='Remove cached data.') _ = parser_clean # unused + # qvm-template repolist + parser_repolist = parser_add_command('repolist', + help_str='Show configured repositories.') + repolim = parser_repolist.add_mutually_exclusive_group() + repolim.add_argument('--all', action='store_true', + help='Show all repos.') + repolim.add_argument('--enabled', action='store_true', + help='Show enabled repos (default).') + repolim.add_argument('--disabled', action='store_true', + help='Show disabled repos.') + parser_repolist.add_argument('repos', nargs='*', metavar='REPOS') return parser_main @@ -679,7 +691,7 @@ def download( os.remove(target_suffix) raise if not done: - print('\'%s\' download failed.' % spec, file=sys.stderr) + print('Error: \'%s\' download failed.' % spec, file=sys.stderr) sys.exit(1) def install( @@ -1108,6 +1120,65 @@ def clean(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: shutil.rmtree(args.cachedir) +def repolist(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: + """Command that lists configured repositories. + + :param args: Arguments received by the application. + :param app: Qubes application object + """ + _ = app # unused + + # python-dnf is not packaged on Debian + # As this is not an "essential operation", the module is imported here + # instead of top-level so that other operations still work. + try: + import dnf + except ModuleNotFoundError: + print("Error: Python module 'dnf' not found.", file=sys.stderr) + sys.exit(1) + + if not args.all and not args.disabled: + args.enabled = True + + with tempfile.TemporaryDirectory(dir=TEMP_DIR) as reposdir: + for idx, path in enumerate(args.repo_files): + src = os.path.abspath(path) + # Use index as file name in case of collisions + dst = os.path.join(reposdir, '%d.repo' % idx) + os.symlink(src, dst) + conf = dnf.conf.Conf() + conf.substitutions['releasever'] = args.releasever + conf.reposdir = reposdir + base = dnf.Base(conf) + base.read_all_repos() + if args.repoid: + base.repos.get_matching('*').disable() + for repo in args.repoid: + base.repos.get_matching(repo).enable() + else: + for repo in args.enablerepo: + base.repos.get_matching(repo).enable() + for repo in args.disablerepo: + base.repos.get_matching(repo).disable() + + if args.repos: + repos = [] + for repo in args.repos: + repos += list(base.repos.get_matching(repo)) + repos = list(set(repos)) + repos.sort(key=operator.attrgetter('id')) + else: + repos = list(base.repos.values()) + repos.sort(key=operator.attrgetter('id')) + + table = [] + for repo in repos: + if args.all or (args.enabled == repo.enabled): + state = 'enabled' if repo.enabled else 'disabled' + table.append((repo.id, repo.name, state)) + + qubesadmin.tools.print_table(table) + def main(args: typing.Optional[typing.Sequence[str]] = None, app: typing.Optional[qubesadmin.app.QubesBase] = None) -> int: """Main routine of **qvm-template**. @@ -1156,6 +1227,8 @@ def main(args: typing.Optional[typing.Sequence[str]] = None, remove(p_args, app) elif p_args.operation == 'clean': clean(p_args, app) + elif p_args.operation == 'repolist': + repolist(p_args, app) else: parser.error('Operation \'%s\' not supported.' % p_args.operation) From 42a741cac559ea5ca3ec858bb63091210cafd699 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Fri, 7 Aug 2020 15:02:40 +0800 Subject: [PATCH 044/119] qvm-template: Remove default 'repo_files' entry if other entries have been specified by the user. --- qubesadmin/tools/qvm_template.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 39f360a..f8fed13 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -1198,6 +1198,11 @@ def main(args: typing.Optional[typing.Sequence[str]] = None, # on inputs where the first argument is an option, like 'qvm-template # remove --help'. The bug should be fixed in Python 3.9. + # If the user specified other repo files... + if len(p_args.repo_files) > 1: + # ...remove the default entry + p_args.repo_files.pop(0) + if app is None: app = qubesadmin.Qubes() From 87c08c994161d21a649ea23ee7e488e06bdc397d Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Fri, 7 Aug 2020 22:58:17 +0800 Subject: [PATCH 045/119] qvm-template: Fix missing args for install operations. --- qubesadmin/tools/qvm_template.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index f8fed13..00b4068 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -71,14 +71,6 @@ def parser_gen() -> argparse.ArgumentParser: help='Set repository metadata as expired before running the command.') parser_main.add_argument('--cachedir', default=CACHE_DIR, help='Specify cache directory.') - # qvm-template download - parser_download = parser_add_command('download', - help_str='Download template package.') - parser_download.add_argument('--downloaddir', default='.', - help='Specify download directory.') - parser_download.add_argument('--retries', default=5, type=int, - help='Specify number of retries for downloads.') - parser_download.add_argument('templates', nargs='*', metavar='TEMPLATE') # qvm-template {install,reinstall,downgrade,upgrade} parser_install = parser_add_command('install', help_str='Install template packages.') @@ -97,6 +89,16 @@ def parser_gen() -> argparse.ArgumentParser: parser_x.add_argument('--allow-pv', action='store_true', help='Allow setting virt_mode to pv in configuration file.') parser_x.add_argument('templates', nargs='*', metavar='TEMPLATE') + # qvm-template download + parser_download = parser_add_command('download', + help_str='Download template package.') + for parser_x in [parser_install, parser_reinstall, + parser_downgrade, parser_upgrade, parser_download]: + parser_x.add_argument('--downloaddir', default='.', + help='Specify download directory.') + parser_x.add_argument('--retries', default=5, type=int, + help='Specify number of retries for downloads.') + parser_download.add_argument('templates', nargs='*', metavar='TEMPLATE') # qvm-template {list,info} parser_list = parser_add_command('list', help_str='List templates.') From 8ee0d639b8adee15fdd3d55176b1e35813235c9e Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 8 Aug 2020 14:39:29 +0800 Subject: [PATCH 046/119] qvm-template: Add confirmation for dangerous operations; verify signatures once instead of twice by returning header after verification. --- qubesadmin/tools/qvm_template.py | 199 +++++++++++++++++-------------- 1 file changed, 111 insertions(+), 88 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 00b4068..9e850d6 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -71,6 +71,8 @@ def parser_gen() -> argparse.ArgumentParser: help='Set repository metadata as expired before running the command.') parser_main.add_argument('--cachedir', default=CACHE_DIR, help='Specify cache directory.') + parser_main.add_argument('--yes', action='store_true', + help='Assume "yes" to questions.') # qvm-template {install,reinstall,downgrade,upgrade} parser_install = parser_add_command('install', help_str='Install template packages.') @@ -486,8 +488,9 @@ def verify_rpm( path: str, nogpgcheck: bool = False, transaction_set: typing.Optional[rpm.transaction.TransactionSet] = None - ) -> bool: - """Verify the digest and signature of a RPM package. + ) -> rpm.hdr: + """Verify the digest and signature of a RPM package and return the package + header. Note that verifying RPMs this way is prone to TOCTOU. This is okay for local files, but may create problems if multiple instances of @@ -498,42 +501,22 @@ def verify_rpm( :param nogpgcheck: Whether to allow invalid GPG signatures :param transaction_set: Override RPM ``TransactionSet``. Optional - :return: Whether the RPM is verified + :return: RPM package header. If verification fails, ``None`` is returned. """ if transaction_set is None: transaction_set = rpm.TransactionSet() with open(path, 'rb') as fd: try: hdr = transaction_set.hdrFromFdno(fd) - if hdr[rpm.RPMTAG_SIGSIZE] is None \ - and hdr[rpm.RPMTAG_SIGPGP] is None \ + if hdr[rpm.RPMTAG_SIGPGP] is None \ and hdr[rpm.RPMTAG_SIGGPG] is None: - return nogpgcheck + return hdr if nogpgcheck else None except rpm.error as e: if str(e) == 'public key not trusted' \ or str(e) == 'public key not available': - return nogpgcheck - return False - return True - -def get_package_hdr( - path: str, - transaction_set: typing.Optional[rpm.transaction.TransactionSet] = None - ) -> rpm.hdr: - """Return header of a RPM package. - - Note that this function **does not** check the integrity of the package. - - :param path: Location of the RPM package - :param transaction_set: Override RPM ``TransactionSet``. Optional - - :return: RPM headers - """ - if transaction_set is None: - transaction_set = rpm.TransactionSet() - with open(path, 'rb') as fd: - hdr = transaction_set.hdrFromFdno(fd) - return hdr + return hdr if nogpgcheck else None + return None + return hdr def extract_rpm(name: str, path: str, target: str) -> bool: """Extract a template RPM package. @@ -724,21 +707,90 @@ def install( try: transaction_set = rpm.TransactionSet() - rpm_list = [] # rpmfile, reponame + unverified_rpm_list = [] # rpmfile, reponame + verified_rpm_list = [] + def verify(rpmfile, reponame): + """Verify package signature and version, remove "unverified" + suffix, and parse package header.""" + if reponame != '@commandline': + path = rpmfile + UNVERIFIED_SUFFIX + else: + path = rpmfile + + package_hdr = verify_rpm(path, args.nogpgcheck, transaction_set) + if not package_hdr: + parser.error('Package \'%s\' verification failed.' % rpmfile) + + if reponame != '@commandline': + os.rename(path, rpmfile) + + package_name = package_hdr[rpm.RPMTAG_NAME] + if not package_name.startswith(PACKAGE_NAME_PREFIX): + parser.error( + 'Illegal package name for package \'%s\'.' % rpmfile) + # Remove prefix to get the real template name + name = package_name[len(PACKAGE_NAME_PREFIX):] + + # Check if already installed + if not override_existing and name in app.domains: + print(('Template \'%s\' already installed, skipping...' + ' (You may want to use the' + ' {reinstall,upgrade,downgrade}' + ' operations.)') % name, file=sys.stderr) + return + + # Check if version is really what we want + if override_existing: + vm = get_managed_template_vm(app, name) + pkg_evr = ( + str(package_hdr[rpm.RPMTAG_EPOCHNUM]), + package_hdr[rpm.RPMTAG_VERSION], + package_hdr[rpm.RPMTAG_RELEASE]) + vm_evr = query_local_evr(vm) + cmp_res = rpm.labelCompare(pkg_evr, vm_evr) + if version_selector == VersionSelector.REINSTALL \ + and cmp_res != 0: + parser.error( + 'Same version of template \'%s\' not found.' \ + % name) + elif version_selector == VersionSelector.LATEST_LOWER \ + and cmp_res != -1: + print(("Template '%s' of lower version" + " already installed, skipping..." % name), + file=sys.stderr) + return + elif version_selector == VersionSelector.LATEST_HIGHER \ + and cmp_res != 1: + print(("Template '%s' of higher version" + " already installed, skipping..." % name), + file=sys.stderr) + return + + verified_rpm_list.append((rpmfile, reponame, name, package_hdr)) + + # Process local templates for template in args.templates: if template.endswith('.rpm'): if not os.path.exists(template): parser.error('RPM file \'%s\' not found.' % template) - rpm_list.append((template, '@commandline')) + unverified_rpm_list.append((template, '@commandline')) + + # First verify local RPMs and extract header + for rpmfile, reponame in unverified_rpm_list: + verify(rpmfile, reponame) + unverified_rpm_list = [] os.makedirs(args.cachedir, exist_ok=True) + # Get list of templates to download dl_list = get_dl_list(args, app, version_selector=version_selector) dl_list_copy = dl_list.copy() - # Verify that the templates are not yet installed for name, entry in dl_list.items(): # Should be ensured by checks in repoquery assert entry.reponame != '@commandline' + # Verify that the templates to be downloaded are not yet installed + # Note that we *still* have to do this again in verify() for + # already-downloaded templates if not override_existing and name in app.domains: print(('Template \'%s\' already installed, skipping...' ' (You may want to use the' @@ -746,75 +798,46 @@ def install( ' operations.)') % name, file=sys.stderr) del dl_list_copy[name] else: + # XXX: Perhaps this is better returned by download() version_str = build_version_str(entry.evr) target_file = \ '%s%s-%s.rpm' % (PACKAGE_NAME_PREFIX, name, version_str) - rpm_list.append( + unverified_rpm_list.append( (os.path.join(args.cachedir, target_file), entry.reponame)) dl_list = dl_list_copy + # Ask the user for confirmation before we actually download stuff + if override_existing and not args.yes: + override_tpls = [] + # Local templates, already verified + for _, _, name, _ in verified_rpm_list: + override_tpls.append(name) + # Templates not yet downloaded + for name in dl_list: + override_tpls.append(name) + + print('This will override changes made in the following VMs:', + file=sys.stderr) + for tpl in override_tpls: + print(' %s' % tpl, file=sys.stderr) + confirm = '' + while confirm != 'y': + confirm = input('Are you sure? [y/N] ').lower() + if confirm == 'n': + sys.exit(1) + download(args, app, path_override=args.cachedir, dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, version_selector=version_selector) - # Verify package and remove unverified suffix - for rpmfile, reponame in rpm_list: - if reponame != '@commandline': - path = rpmfile + UNVERIFIED_SUFFIX - else: - path = rpmfile - if not verify_rpm(path, args.nogpgcheck, transaction_set): - parser.error('Package \'%s\' verification failed.' % rpmfile) - if reponame != '@commandline': - os.rename(path, rpmfile) + # Verify downloaded templates + for rpmfile, reponame in unverified_rpm_list: + verify(rpmfile, reponame) + unverified_rpm_list = [] # Unpack and install - for rpmfile, reponame in rpm_list: + for rpmfile, reponame, name, package_hdr in verified_rpm_list: with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: - package_hdr = get_package_hdr(rpmfile) - package_name = package_hdr[rpm.RPMTAG_NAME] - if not package_name.startswith(PACKAGE_NAME_PREFIX): - parser.error( - 'Illegal package name for package \'%s\'.' % rpmfile) - # Remove prefix to get the real template name - name = package_name[len(PACKAGE_NAME_PREFIX):] - - # Another check for already-downloaded RPMs - if not override_existing and name in app.domains: - print(('Template \'%s\' already installed, skipping...' - ' (You may want to use the' - ' {reinstall,upgrade,downgrade}' - ' operations.)') % name, file=sys.stderr) - continue - - # Check if local versus candidate version is in line with the - # operation - if override_existing: - vm = get_managed_template_vm(app, name) - pkg_evr = ( - str(package_hdr[rpm.RPMTAG_EPOCHNUM]), - package_hdr[rpm.RPMTAG_VERSION], - package_hdr[rpm.RPMTAG_RELEASE]) - vm_evr = query_local_evr(vm) - cmp_res = rpm.labelCompare(pkg_evr, vm_evr) - if version_selector == VersionSelector.REINSTALL \ - and cmp_res != 0: - parser.error( - 'Same version of template \'%s\' not found.' \ - % name) - elif version_selector == VersionSelector.LATEST_LOWER \ - and cmp_res != -1: - print(("Template '%s' of lower version" - " already installed, skipping..." % name), - file=sys.stderr) - continue - elif version_selector == VersionSelector.LATEST_HIGHER \ - and cmp_res != 1: - print(("Template '%s' of higher version" - " already installed, skipping..." % name), - file=sys.stderr) - continue - print('Installing template \'%s\'...' % name, file=sys.stderr) extract_rpm(name, rpmfile, target) cmdline = [ @@ -824,7 +847,7 @@ def install( ] if args.allow_pv: cmdline.append('--allow-pv') - if args.pool: + if not override_existing and args.pool: cmdline += ['--pool', args.pool] subprocess.check_call(cmdline + [ 'post-install', From 6c873cdf3981c33fd87a251d49311100468e3b19 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 8 Aug 2020 14:57:22 +0800 Subject: [PATCH 047/119] qvm-template-postprocess: Make pylint happy. --- qubesadmin/tools/qvm_template_postprocess.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py index c6b51f7..36bbcdd 100644 --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -161,15 +161,15 @@ def import_appmenus(vm, source_dir): # separated by spaces should be ok as there should be no spaces in the file # name according to the FreeDesktop spec with open(os.path.join(source_dir, 'vm-whitelisted-appmenus.list'), 'r') \ - as f: - vm.features['default-menu-items'] = ' '.join([x.rstrip() for x in f]) + as fd: + vm.features['default-menu-items'] = ' '.join([x.rstrip() for x in fd]) with open(os.path.join(source_dir, 'whitelisted-appmenus.list'), 'r') \ - as f: - vm.features['menu-items'] = ' '.join([x.rstrip() for x in f]) + as fd: + vm.features['menu-items'] = ' '.join([x.rstrip() for x in fd]) with open( os.path.join(source_dir, 'netvm-whitelisted-appmenus.list'), 'r') \ - as f: - vm.features['netvm-menu-items'] = ' '.join([x.rstrip() for x in f]) + as fd: + vm.features['netvm-menu-items'] = ' '.join([x.rstrip() for x in fd]) # TODO: change this to qrexec calls to GUI VM, when GUI VM will be # implemented @@ -185,8 +185,8 @@ def import_appmenus(vm, source_dir): def parse_template_config(path): '''Parse template.conf from template package. (KEY=VALUE format)''' - with open(path, 'r') as f: - return dict(line.rstrip('\n').split('=', 1) for line in f) + with open(path, 'r') as fd: + return dict(line.rstrip('\n').split('=', 1) for line in fd) @asyncio.coroutine def call_postinstall_service(vm): From ed8fca6494f5f2b43c8eb79c4720e354454bbd14 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 8 Aug 2020 15:31:25 +0800 Subject: [PATCH 048/119] qvm-template: Fix type hints. --- qubesadmin/tools/qvm_template.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 9e850d6..cbcf158 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -818,8 +818,8 @@ def install( print('This will override changes made in the following VMs:', file=sys.stderr) - for tpl in override_tpls: - print(' %s' % tpl, file=sys.stderr) + for name in override_tpls: + print(' %s' % name, file=sys.stderr) confirm = '' while confirm != 'y': confirm = input('Are you sure? [y/N] ').lower() @@ -1186,6 +1186,7 @@ def repolist(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: for repo in args.disablerepo: base.repos.get_matching(repo).disable() + repos: typing.List[dnf.repo.Repo] if args.repos: repos = [] for repo in args.repos: From b7a603b9fee8ea4211f6cbc8d50ce720b7d510d0 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Mon, 10 Aug 2020 01:30:31 +0800 Subject: [PATCH 049/119] qvm-template: Slight improvements to package verification. --- qubesadmin/tools/qvm_template.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index cbcf158..dc68ed4 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -721,9 +721,6 @@ def install( if not package_hdr: parser.error('Package \'%s\' verification failed.' % rpmfile) - if reponame != '@commandline': - os.rename(path, rpmfile) - package_name = package_hdr[rpm.RPMTAG_NAME] if not package_name.startswith(PACKAGE_NAME_PREFIX): parser.error( @@ -731,6 +728,9 @@ def install( # Remove prefix to get the real template name name = package_name[len(PACKAGE_NAME_PREFIX):] + if path != rpmfile: + os.rename(path, rpmfile) + # Check if already installed if not override_existing and name in app.domains: print(('Template \'%s\' already installed, skipping...' From 3314500a83be66d46bb00cd039bb54322fb3f65f Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Fri, 14 Aug 2020 11:38:30 +0800 Subject: [PATCH 050/119] qvm-template: Add purge operation. --- qubesadmin/tools/qvm_template.py | 139 +++++++++++++++++++++++-------- 1 file changed, 106 insertions(+), 33 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index dc68ed4..87e9e91 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -21,6 +21,8 @@ import typing import qubesadmin import qubesadmin.tools +import qubesadmin.tools.qvm_kill +import qubesadmin.tools.qvm_remove import rpm import tqdm import xdg.BaseDirectory @@ -49,9 +51,12 @@ def parser_gen() -> argparse.ArgumentParser: subparsers = parser_main.add_subparsers(dest='operation', required=True, description='Command to run.') - def parser_add_command(cmd, help_str, add_help=True): - return subparsers.add_parser(cmd, formatter_class=formatter, - help=help_str, description=help_str, add_help=add_help) + def parser_add_command(cmd, help_str): + return subparsers.add_parser( + cmd, + formatter_class=formatter, + help=help_str, + description=help_str) # qrexec/DNF related parser_main.add_argument('--repo-files', action='append', @@ -130,9 +135,14 @@ def parser_gen() -> argparse.ArgumentParser: parser_search.add_argument('templates', nargs='*', metavar='PATTERN') # qvm-template remove parser_remove = parser_add_command('remove', - help_str='Remove installed templates.', - add_help=False) # Forward --help to qvm-remove - _ = parser_remove # unused + help_str='Remove installed templates.') + parser_remove.add_argument('--disassoc', action='store_true', + help='Also disassociate VMs from the templates to be removed.') + parser_remove.add_argument('templates', nargs='*', metavar='TEMPLATE') + # qvm-template purge + parser_purge = parser_add_command('purge', + help_str='Remove installed templates and associated VMs.') + parser_purge.add_argument('templates', nargs='*', metavar='TEMPLATE') # qvm-template clean parser_clean = parser_add_command('clean', help_str='Remove cached data.') @@ -282,6 +292,19 @@ def get_managed_template_vm(app: qubesadmin.app.QubesBase, name: str parser.error("Template '%s' is not managed by qvm-template." % name) return vm +def confirm_action(msg: str, affected: typing.List[str]) -> None: + """Confirm user action.""" + print(msg) + for name in affected: + print(' ' + name) + + confirm = '' + while confirm != 'y': + confirm = input('Are you sure? [y/N] ').lower() + if confirm == 'n': + print('Operation cancelled.') + sys.exit(1) + def qrexec_popen( args: argparse.Namespace, app: qubesadmin.app.QubesBase, @@ -816,15 +839,9 @@ def install( for name in dl_list: override_tpls.append(name) - print('This will override changes made in the following VMs:', - file=sys.stderr) - for name in override_tpls: - print(' %s' % name, file=sys.stderr) - confirm = '' - while confirm != 'y': - confirm = input('Are you sure? [y/N] ').lower() - if confirm == 'n': - sys.exit(1) + confirm_action( + 'This will override changes made in the following VMs:', + override_tpls) download(args, app, path_override=args.cachedir, dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, @@ -1118,21 +1135,83 @@ def search(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: print('===', cur_header, '===') print(query_res[idx].name, ':', query_res[idx].summary) -def remove(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: +def remove( + args: argparse.Namespace, + app: qubesadmin.app.QubesBase, + disassoc: bool = False, + purge: bool = False, + dummy: str = 'dummy' + ) -> None: """Command that remove templates. :param args: Arguments received by the application. :param app: Qubes application object + :param disassoc: Whether to disassociate VMs from the templates + :param purge: Whether to remove VMs based on the templates + :param dummy: Name of dummy VM if disassoc is used """ - _ = args, app # unused + # NOTE: While QubesArgumentParser provide similar functionality + # it does not seem to work as a parent parser + for tpl in args.templates: + if tpl not in app.domains: + parser.error("no such domain: '%s'" % tpl) - # Remove 'remove' entry from the args... - operation_idx = sys.argv.index('remove') - argv = sys.argv[1:operation_idx] + sys.argv[operation_idx+1:] + remove_list = args.templates + if purge: + # Not disassociating first may result in dependency ordering issues + disassoc = True + # Remove recursively via BFS + remove_set = set(remove_list) # visited + idx = 0 + while idx < len(remove_list): + tpl = remove_list[idx] + idx += 1 + vm = app.domains[tpl] + for holder, prop in qubesadmin.utils.vm_dependencies(app, vm): + if holder is not None and holder.name not in remove_set: + remove_list.append(holder.name) + remove_set.add(holder.name) - # ...then pass the args to qvm-remove - # Use exec so stdio can be shared easily - os.execvp('qvm-remove', ['qvm-remove'] + argv) + if not args.yes: + repeat = 3 if purge else 1 + for _ in range(repeat): + confirm_action( + 'This will completely remove the selected VM(s)...', + remove_list) + + if disassoc: + # Remove the dummy afterwards if we're purging + remove_dummy = purge + # Create dummy template; handle name collisions + orig_dummy = dummy + cnt = 1 + while dummy in app.domains \ + and not app.domains[dummy].features.get('template-dummy', 0): + dummy = '%s-%d' % (orig_dummy, cnt) + cnt += 1 + if dummy not in app.domains: + dummy_vm = app.add_new_vm('TemplateVM', dummy, 'red') + else: + dummy_vm = app.domains[dummy] + + for tpl in remove_list: + vm = app.domains[tpl] + for holder, prop in qubesadmin.utils.vm_dependencies(app, vm): + if holder: + setattr(holder, prop, dummy_vm) + holder.template = dummy_vm + print("Property '%s' of '%s' set to '%s'." % ( + prop, holder.name, dummy), file=sys.stderr) + else: + print("Global property '%s' set to ''." % prop, + file=sys.stderr) + setattr(app, prop, '') + if remove_dummy: + remove_list.append(dummy) + + if disassoc or purge: + qubesadmin.tools.qvm_kill.main(['--'] + remove_list, app) + qubesadmin.tools.qvm_remove.main(['--force', '--'] + remove_list, app) def clean(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: """Command that cleans the local package cache. @@ -1214,15 +1293,7 @@ def main(args: typing.Optional[typing.Sequence[str]] = None, :return: Return code of the application """ - p_args, unk_args = parser.parse_known_args(args) - if p_args.operation != 'remove' and unk_args: - p_args = parser.parse_args(args) # this should result in an error - assert False and 'This line should not be executed.' - # FIXME: Currently doing things this way as we have to forward - # arguments to qvm-remove. While argparse.REMAINDER should be able to - # solve this, there's a bug (issue 17050) that prevents it from working - # on inputs where the first argument is an option, like 'qvm-template - # remove --help'. The bug should be fixed in Python 3.9. + p_args = parser.parse_args(args) # If the user specified other repo files... if len(p_args.repo_files) > 1: @@ -1255,7 +1326,9 @@ def main(args: typing.Optional[typing.Sequence[str]] = None, elif p_args.operation == 'search': search(p_args, app) elif p_args.operation == 'remove': - remove(p_args, app) + remove(p_args, app, disassoc=p_args.disassoc) + elif p_args.operation == 'purge': + remove(p_args, app, purge=True) elif p_args.operation == 'clean': clean(p_args, app) elif p_args.operation == 'repolist': From c6d5ac7c8c1104008126b3064913d1fdd55479ac Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Fri, 14 Aug 2020 14:27:36 +0800 Subject: [PATCH 051/119] qvm-template: Add option to specify RPM keyring location. --- qubesadmin/tools/qvm_template.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 87e9e91..047b26e 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -62,6 +62,9 @@ def parser_gen() -> argparse.ArgumentParser: parser_main.add_argument('--repo-files', action='append', default=['/usr/share/qubes/repo-templates/qubes-templates.repo'], help='Specify files containing DNF repository configuration.') + parser_main.add_argument('--keyring', + default='/usr/share/qubes/repo-templates/keys', + help='Specify directory containing RPM public keys.') parser_main.add_argument('--updatevm', default='sys-firewall', help='Specify VM to download updates from.') parser_main.add_argument('--enablerepo', action='append', default=[], @@ -507,10 +510,22 @@ def qrexec_download( raise ConnectionError( "qrexec call 'qubes.TemplateDownload' failed.") +def rpm_transactionset(key_dir: str) -> rpm.transaction.TransactionSet: + """Create RPM TransactionSet using the keys in the given directory.""" + tset = rpm.TransactionSet() + kring = rpm.keyring() + for name in os.listdir(key_dir): + path = os.path.join(key_dir, name) + if os.path.isfile(path): + with open(path, 'rb') as fd: + kring.addKey(rpm.pubkey(fd.read())) + tset.setKeyring(kring) + return tset + def verify_rpm( path: str, - nogpgcheck: bool = False, - transaction_set: typing.Optional[rpm.transaction.TransactionSet] = None + transaction_set: rpm.transaction.TransactionSet, + nogpgcheck: bool = False ) -> rpm.hdr: """Verify the digest and signature of a RPM package and return the package header. @@ -521,13 +536,11 @@ def verify_rpm( case. :param path: Location of the RPM package + :param transaction_set: RPM ``TransactionSet`` :param nogpgcheck: Whether to allow invalid GPG signatures - :param transaction_set: Override RPM ``TransactionSet``. Optional :return: RPM package header. If verification fails, ``None`` is returned. """ - if transaction_set is None: - transaction_set = rpm.TransactionSet() with open(path, 'rb') as fd: try: hdr = transaction_set.hdrFromFdno(fd) @@ -728,7 +741,7 @@ def install( % LOCK_FILE) try: - transaction_set = rpm.TransactionSet() + transaction_set = rpm_transactionset(args.keyring) unverified_rpm_list = [] # rpmfile, reponame verified_rpm_list = [] @@ -740,7 +753,7 @@ def install( else: path = rpmfile - package_hdr = verify_rpm(path, args.nogpgcheck, transaction_set) + package_hdr = verify_rpm(path, transaction_set, args.nogpgcheck) if not package_hdr: parser.error('Package \'%s\' verification failed.' % rpmfile) From d09695658f1bda30a3d6bac6856af39aed7c781e Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Wed, 19 Aug 2020 01:59:51 +0800 Subject: [PATCH 052/119] qvm-template: Add support for JSON output. --- qubesadmin/tools/qvm_template.py | 74 ++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 047b26e..2774552 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -9,6 +9,7 @@ import enum import fnmatch import functools import itertools +import json import operator import os import re @@ -58,7 +59,6 @@ def parser_gen() -> argparse.ArgumentParser: help=help_str, description=help_str) - # qrexec/DNF related parser_main.add_argument('--repo-files', action='append', default=['/usr/share/qubes/repo-templates/qubes-templates.repo'], help='Specify files containing DNF repository configuration.') @@ -126,8 +126,11 @@ def parser_gen() -> argparse.ArgumentParser: ' locally but not in repos) templates.')) parser_x.add_argument('--upgrades', action='store_true', help='Show upgradable templates.') - parser_x.add_argument('--machine-readable', action='store_true', + readable = parser_x.add_mutually_exclusive_group() + readable.add_argument('--machine-readable', action='store_true', help='Enable machine-readable output.') + readable.add_argument('--machine-readable-json', action='store_true', + help='Enable machine-readable output (JSON).') parser_x.add_argument('templates', nargs='*', metavar='TEMPLATE') # qvm-template search parser_search = parser_add_command('search', @@ -932,10 +935,21 @@ def list_templates(args: argparse.Namespace, def append_info(data, status, install_time=None): tpl_list.append((status, data, install_time)) - def list_to_output(tpls): + def list_to_human_output(tpls): outputs = [] for status, grp in itertools.groupby(tpls, lambda x: x[0]): - outputs.append((status, list(map(lambda x: x[1:], grp)))) + def convert(row): + return row[1:] + outputs.append((status, list(map(convert, grp)))) + return outputs + + def list_to_machine_output(tpls): + outputs = {} + for status, grp in itertools.groupby(tpls, lambda x: x[0]): + def convert(row): + _, name, evr, reponame = row + return {'name': name, 'evr': evr, 'reponame': reponame} + outputs[status.value] = list(map(convert, grp)) return outputs def info_to_human_output(tpls): @@ -966,7 +980,7 @@ def list_templates(args: argparse.Namespace, return outputs def info_to_machine_output(tpls, replace_newline=True): - outputs = [] + outputs = {} for status, grp in itertools.groupby(tpls, lambda x: x[0]): output = [] for _, data, install_time in grp: @@ -977,10 +991,20 @@ def list_templates(args: argparse.Namespace, install_time = str(install_time) if install_time else '' if replace_newline: description = description.replace('\n', '|') - output.append((name, epoch, version, release, reponame, - dlsize, buildtime, install_time, licence, url, summary, - description)) - outputs.append((status, output)) + output.append({ + 'name': name, + 'epoch': epoch, + 'version': version, + 'release': release, + 'reponame': reponame, + 'size': dlsize, + 'buildtime': buildtime, + 'installtime': install_time, + 'license': licence, + 'url': url, + 'summary': summary, + 'description': description}) + outputs[status.value] = output return outputs if operation == 'list': @@ -1042,23 +1066,29 @@ def list_templates(args: argparse.Namespace, if len(tpl_list) == 0: parser.error('No matching templates to list') - if not args.machine_readable: - if operation == 'info': - tpl_list = info_to_human_output(tpl_list) - elif operation == 'list': - tpl_list = list_to_output(tpl_list) - for status, grp in tpl_list: - print(status.title()) - qubesadmin.tools.print_table(grp) - else: + if args.machine_readable: if operation == 'info': tpl_list = info_to_machine_output(tpl_list) elif operation == 'list': - tpl_list = list_to_output(tpl_list) - for status, grp in tpl_list: - print('|' + status.value) + tpl_list = list_to_machine_output(tpl_list) + for status, grp in tpl_list.items(): + print('|' + status) for line in grp: - print('|'.join(line) + '|') + print('|'.join(line.values()) + '|') + elif args.machine_readable_json: + if operation == 'info': + tpl_list = info_to_machine_output(tpl_list, replace_newline=False) + elif operation == 'list': + tpl_list = list_to_machine_output(tpl_list) + print(json.dumps(tpl_list)) + else: + if operation == 'info': + tpl_list = info_to_human_output(tpl_list) + elif operation == 'list': + tpl_list = list_to_human_output(tpl_list) + for status, grp in tpl_list: + print(status.title()) + qubesadmin.tools.print_table(grp) def search(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: """Command that searches template details for given patterns. From 55a3982bf64981222aba5bfcaf8287f6c2b36f13 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Wed, 19 Aug 2020 02:00:19 +0800 Subject: [PATCH 053/119] qvm-template: Add option to disable download progress bar. --- qubesadmin/tools/qvm_template.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 2774552..8e5cea8 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -81,6 +81,8 @@ def parser_gen() -> argparse.ArgumentParser: help='Specify cache directory.') parser_main.add_argument('--yes', action='store_true', help='Assume "yes" to questions.') + parser_main.add_argument('--quiet', action='store_true', + help='Reduce amount of output.') # qvm-template {install,reinstall,downgrade,upgrade} parser_install = parser_add_command('install', help_str='Install template packages.') @@ -482,8 +484,8 @@ def qrexec_download( """Download a template from repositories. :param args: Arguments received by the application. Specifically, - ``args.{enablerepo,disablerepo,repoid,releasever,repo_files,updatevm}`` - are used + ``args.{enablerepo,disablerepo,repoid,releasever,repo_files,updatevm, + quiet}`` are used :param app: Qubes application object :param spec: Package spec to query (refer to ```` in the DNF documentation) @@ -502,7 +504,7 @@ def qrexec_download( proc.stdin.write(payload.encode('UTF-8')) proc.stdin.close() with tqdm.tqdm(desc=spec, total=dlsize, unit_scale=True, - unit_divisor=1000, unit='B') as pbar: + unit_divisor=1000, unit='B', disable=args.quiet) as pbar: last = 0 while proc.poll() is None: cur = fd.tell() From e9e198cc10c656dae17c5f983413bc2aa7660385 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 25 Aug 2020 01:43:11 +0800 Subject: [PATCH 054/119] qvm-template: Make sure that template-dummy is set and used properly. --- qubesadmin/tools/qvm_template.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 8e5cea8..3558a9d 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -1231,11 +1231,13 @@ def remove( orig_dummy = dummy cnt = 1 while dummy in app.domains \ - and not app.domains[dummy].features.get('template-dummy', 0): + and app.domains[dummy].features.get( + 'template-dummy', '0') == '0': dummy = '%s-%d' % (orig_dummy, cnt) cnt += 1 if dummy not in app.domains: dummy_vm = app.add_new_vm('TemplateVM', dummy, 'red') + dummy_vm.features['template-dummy'] = 1 else: dummy_vm = app.domains[dummy] From 6efd85afbab45278f78f263d67daffb95de99970 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 25 Aug 2020 23:00:08 +0800 Subject: [PATCH 055/119] qvm-template: Initial manpage. --- doc/manpages/qvm-template.rst | 499 +++++++++++++++++++++++++++++++ qubesadmin/tools/qvm_template.py | 44 +-- 2 files changed, 525 insertions(+), 18 deletions(-) create mode 100644 doc/manpages/qvm-template.rst diff --git a/doc/manpages/qvm-template.rst b/doc/manpages/qvm-template.rst new file mode 100644 index 0000000..2b0c50b --- /dev/null +++ b/doc/manpages/qvm-template.rst @@ -0,0 +1,499 @@ +.. program:: qvm-template + +:program:`qvm-template` -- Manage template VMs +============================================== + +Synopsis +-------- + +:command:`qvm-template` [-h] [--repo-files *REPO_FILES*] [--keyring *KEYRING*] [--updatevm *UPDATEVM*] [--enablerepo *REPOID*] [--disablerepo *REPOID*] [--repoid *REPOID*] [--releasever *RELEASEVER*] [--refresh] [--cachedir *CACHEDIR*] [--yes] [--quiet] *SUBCOMMAND* + +See Section `Subcommands`_ for available subcommands. + +Options +------- + +.. option:: --help, -h + + Show help message and exit. + +.. option:: --repo-files REPO_FILES + + Specify files containing DNF repository configuration. Can be + used more than once. (default: + ['/usr/share/qubes/repo-templates/qubes-templates.repo']) + +.. option:: --keyring KEYRING + + Specify directory containing RPM public keys. (default: + /usr/share/qubes/repo-templates/keys) + +.. option:: --updatevm UPDATEVM + + Specify VM to download updates from. (default: sys-firewall) + +.. option:: --enablerepo REPOID + + Enable additional repositories by an id or a glob. Can be used more than + once. + +.. option:: --disablerepo REPOID + + Disable certain repositories by an id or a glob. Can be used more than once. + +.. option:: --repoid REPOID + + Enable just specific repositories by an id or a glob. Can be used more than + once. + +.. option:: --releasever RELEASEVER + + Override Qubes release version. + +.. option:: --refresh + + Set repository metadata as expired before running the command. + +.. option:: --cachedir CACHEDIR + + Specify cache directory. (default: ~/.cache/qvm-template) + +.. option:: --yes + + Assume "yes" to questions. + +.. option:: --quiet + + Decrease verbosity. + +Subcommands +=========== + +install +------- + +Install template packages. + +Synopsis +^^^^^^^^ + +:command:`qvm-template install` [-h] [--pool *POOL*] [--nogpgcheck] [--allow-pv] [--downloaddir *DOWNLOADDIR*] [--retries *RETRIES*] [*TEMPLATESPEC* [*TEMPLATESPEC* ...]] + +See Section `Template Spec`_ for an explanation of *TEMPLATESPEC*. + +Options +^^^^^^^ + +.. option:: -h, --help + + Show help message and exit. + +.. option:: --pool POOL + + Specify pool to store created VMs in. + +.. option:: --nogpgcheck + + Disable signature checks. + +.. option:: --allow-pv + + Allow templates that set virt_mode to pv. + +.. option:: --downloaddir DOWNLOADDIR + + Specify download directory. (default: .) + +.. option:: --retries RETRIES + + Specify maximum number of retries for downloads. (default: 5) + +{reinstall,downgrade,upgrade} +----------------------------- + +Reinstall/downgrade/upgrade template packages. + +Synopsis +^^^^^^^^ + +:command:`qvm-template {reinstall,downgrade,upgrade}` [-h] [--nogpgcheck] [--allow-pv] [--downloaddir *DOWNLOADDIR*] [--retries *RETRIES*] [*TEMPLATESPEC* [*TEMPLATESPEC* ...]] + +See Section `Template Spec`_ for an explanation of *TEMPLATESPEC*. + +Options +^^^^^^^ + +.. option:: -h, --help + + Show help message and exit. + +.. option:: --nogpgcheck + + Disable signature checks. + +.. option:: --allow-pv + + Allow templates that set virt_mode to pv. + +.. option:: --downloaddir DOWNLOADDIR + + Specify download directory. (default: .) + +.. option:: --retries RETRIES + + Specify maximum number of retries for downloads. (default: 5) + +download +-------- + +Download template packages. + +Synopsis +^^^^^^^^ + +:command:`qvm-template download` [-h] [--downloaddir *DOWNLOADDIR*] [--retries *RETRIES*] [*TEMPLATESPEC* [*TEMPLATESPEC* ...]] + +See Section `Template Spec`_ for an explanation of *TEMPLATESPEC*. + +Options +^^^^^^^ + +.. option:: -h, --help + + Show help message and exit. + +.. option:: --downloaddir DOWNLOADDIR + + Specify download directory. (default: .) + +.. option:: --retries RETRIES + + Specify maximum number of retries for downloads. (default: 5) + +list +---- + +List templates. + +Synopsis +^^^^^^^^ + +:command:`qvm-template list` [-h] [--all] [--installed] [--available] [--extras] [--upgrades] [--machine-readable | --machine-readable-json] [*TEMPLATESPEC* [*TEMPLATESPEC* ...]] + +See Section `Template Spec`_ for an explanation of *TEMPLATESPEC*. + +Options +^^^^^^^ + +.. option:: -h, --help + + Show help message and exit. + +.. option:: --all + + Show all templates (default). + +.. option:: --installed + + Show installed templates. + +.. option:: --available + + Show available templates. + +.. option:: --extras + + Show extras (e.g., ones that exist locally but not in repos) + templates. + +.. option:: --upgrades + + Show upgradable templates. + +.. option:: --machine-readable + + Enable machine-readable output. + + Format + Each line describes a template in the following format: + + :: + + |{status}|{name}|{evr}|{reponame}| + + Where ``{status}`` can be one of ``installed``, ``available``, + ``extra``, or ``upgradable``. + + The field ``{evr}`` contains version information in the form of + ``{epoch}:{version}-{release}``. + +.. option:: --machine-readable-json + + Enable machine-readable output (JSON). + + Format + The resulting JSON document is in the following format: + + :: + + { + STATUS: [ + { + "name": str, + "evr": str, + "reponame": str + }, + ... + ], + ... + } + + Where ``STATUS`` can be one of ``"installed"``, ``"available"``, + ``"extra"``, or ``"upgradable"``. + + The fields ``buildtime`` and ``installtime`` are in ISO 8601 format. + For example, one can parse them in Python with + ``datetime.fromisoformat()``. + + The field ``{evr}`` contains version information in the form of + ``{epoch}:{version}-{release}``. + +info +---- + +Display details about templates. + +Synopsis +^^^^^^^^ + +:command:`qvm-template list` [-h] [--all] [--installed] [--available] [--extras] [--upgrades] [--machine-readable | --machine-readable-json] [*TEMPLATESPEC* [*TEMPLATESPEC* ...]] + +See Section `Template Spec`_ for an explanation of *TEMPLATESPEC*. + +Options +^^^^^^^ + +.. option:: -h, --help + + Show help message and exit. + +.. option:: --all + + Show all templates (default). + +.. option:: --installed + + Show installed templates. + +.. option:: --available + + Show available templates. + +.. option:: --extras + + Show extras (e.g., ones that exist locally but not in repos) + templates. + +.. option:: --upgrades + + Show upgradable templates. + +.. option:: --machine-readable + + Enable machine-readable output. + + Format + Each line describes a template in the following format: + + :: + + |{status}|{name}|{epoch}|{version}|{release}|{reponame}|{size}|{buildtime}|{installtime}|{license}|{url}|{summary}|{description}| + + Where ``{status}`` can be one of ``installed``, ``available``, + ``extra``, or ``upgradable``. + + The fields ``{buildtime}`` and ``{installtime}`` are in ISO 8601 format. + For example, one can parse them in Python with + ``datetime.fromisoformat()``. + + Newlines in the ``{description}`` field are replaced with pipe + characters (``|``) for easier processing. + +.. option:: --machine-readable-json + + Enable machine-readable output (JSON). + + Format + The resulting JSON document is in the following format: + + :: + + { + STATUS: [ + { + "name": str, + "epoch": str, + "version": str, + "release": str, + "reponame": str, + "size": int, + "buildtime": str, + "installtime": str, + "license": str, + "url": str, + "summary": str, + "description": str + }, + ... + ], + ... + } + + Where ``STATUS`` can be one of ``"installed"``, ``"available"``, + ``"extra"``, or ``"upgradable"``. + + The fields ``buildtime`` and ``installtime`` are in ISO 8601 format. + For example, one can parse them in Python using + `datetime.fromisoformat()`. + +search +------ + +Search template details for the given string. + +Synopsis +^^^^^^^^ + +:command:`qvm-template search` [-h] [--all] [*PATTERN* [*PATTERN* ...]] + +Options +^^^^^^^ + +.. option:: -h, --help + + Show help message and exit. + +.. option:: --all + + Search also in the template description and URL. In addition, the criterion + are evaluated with OR instead of AND. + +remove +------ + +Remove installed templates. + +Synopsis +^^^^^^^^ + +:command:`qvm-template remove` [-h] [--disassoc] [*TEMPLATE* [*TEMPLATE* ...]] + +Options +^^^^^^^ + +.. option:: -h, --help + + Show help message and exit. + +.. option:: --disassoc + + Also disassociate VMs from the templates to be removed. This + creates a *dummy* template for the VMs to link with. + +purge +----- + +Remove installed templates and associated VMs. + +Synopsis +^^^^^^^^ + +:command:`qvm-template purge` [-h] [*TEMPLATE* [*TEMPLATE* ...]] + +Options +^^^^^^^ + +.. option:: -h, --help + + Show help message and exit. + +clean +----- + +Remove locally cached packages. + +Synopsis +^^^^^^^^ + +:command:`qvm-template clean` [-h] + +Options +^^^^^^^ + +.. option:: -h, --help + + Show help message and exit. + +repolist +-------- + +Show configured repositories. + +Synopsis +^^^^^^^^ + +:command:`qvm-template repolist` [-h] [--all | --enabled | --disabled] [*REPOS* [*REPOS* ...]] + +Options +^^^^^^^ + +.. option:: -h, --help + + Show help message and exit. + +.. option:: --all + + Show all repos. + +.. option:: --enabled + + Show only enabled repos (default). + +.. option:: --disabled + + Show only disabled repos. + +Template Spec +------------- + +Subcommands such as ``install`` and ``download`` accept one or more +*TEMPLATESPEC* strings. The format is, in essence, almost identical to +```` described in the DNF documentation. + +In short, the spec is matched against the following list of NEVRA forms, in +decreasing orders of priority: + +* ``name-[epoch:]version-release`` +* ``name`` +* ``name-[epoch:]version`` + +Note that unlike DNF, ``arch`` is currently ignored as the template packages +should all be of ``noarch``. + +One can also use globs in spec strings. See Section `Globs`_ for details. + +Refer to Section *NEVRA Matching* in the DNF documentation for details. + +Globs +----- + +`Template Spec`_ strings, repo ids, and search patterns support glob pattern +matching. In particular, the following special characters can be used: + +* ``*``: Matches any number of characters. +* ``?``: Matches exactly one character. +* ``[]``: Matches any enclosed character. +* ``[!]``: Matches any character except those enclosed. + +In particular, note that ``{}``, while supported by DNF, is not supported by +`qvm-template`. diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 3558a9d..6173623 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -61,20 +61,26 @@ def parser_gen() -> argparse.ArgumentParser: parser_main.add_argument('--repo-files', action='append', default=['/usr/share/qubes/repo-templates/qubes-templates.repo'], - help='Specify files containing DNF repository configuration.') + help=('Specify files containing DNF repository configuration.' + ' Can be used more than once.')) parser_main.add_argument('--keyring', default='/usr/share/qubes/repo-templates/keys', help='Specify directory containing RPM public keys.') parser_main.add_argument('--updatevm', default='sys-firewall', help='Specify VM to download updates from.') parser_main.add_argument('--enablerepo', action='append', default=[], - help='Enable additional repositories.') + metavar='REPOID', + help=('Enable additional repositories by an id or a glob.' + ' Can be used more than once.')) parser_main.add_argument('--disablerepo', action='append', default=[], - help='Disable certain repositories.') + metavar='REPOID', + help=('Disable certain repositories by an id or a glob.' + ' Can be used more than once.')) parser_main.add_argument('--repoid', action='append', default=[], - help='Enable just specific repositories.') + help=('Enable just specific repositories by an id or a glob.' + ' Can be used more than once.')) parser_main.add_argument('--releasever', default=qubes_release(), - help='Override distro release version.') + help='Override Qubes release version.') parser_main.add_argument('--refresh', action='store_true', help='Set repository metadata as expired before running the command.') parser_main.add_argument('--cachedir', default=CACHE_DIR, @@ -82,7 +88,7 @@ def parser_gen() -> argparse.ArgumentParser: parser_main.add_argument('--yes', action='store_true', help='Assume "yes" to questions.') parser_main.add_argument('--quiet', action='store_true', - help='Reduce amount of output.') + help='Decrease verbosity.') # qvm-template {install,reinstall,downgrade,upgrade} parser_install = parser_add_command('install', help_str='Install template packages.') @@ -99,18 +105,19 @@ def parser_gen() -> argparse.ArgumentParser: parser_x.add_argument('--nogpgcheck', action='store_true', help='Disable signature checks.') parser_x.add_argument('--allow-pv', action='store_true', - help='Allow setting virt_mode to pv in configuration file.') - parser_x.add_argument('templates', nargs='*', metavar='TEMPLATE') + help='Allow templates that set virt_mode to pv.') + parser_x.add_argument('templates', nargs='*', metavar='TEMPLATESPEC') # qvm-template download parser_download = parser_add_command('download', - help_str='Download template package.') + help_str='Download template packages.') for parser_x in [parser_install, parser_reinstall, parser_downgrade, parser_upgrade, parser_download]: parser_x.add_argument('--downloaddir', default='.', help='Specify download directory.') parser_x.add_argument('--retries', default=5, type=int, - help='Specify number of retries for downloads.') - parser_download.add_argument('templates', nargs='*', metavar='TEMPLATE') + help='Specify maximum number of retries for downloads.') + parser_download.add_argument('templates', nargs='*', + metavar='TEMPLATESPEC') # qvm-template {list,info} parser_list = parser_add_command('list', help_str='List templates.') @@ -124,7 +131,7 @@ def parser_gen() -> argparse.ArgumentParser: parser_x.add_argument('--available', action='store_true', help='Show available templates.') parser_x.add_argument('--extras', action='store_true', - help=('Show extras (e.g., ones that exists' + help=('Show extras (e.g., ones that exist' ' locally but not in repos) templates.')) parser_x.add_argument('--upgrades', action='store_true', help='Show upgradable templates.') @@ -133,19 +140,20 @@ def parser_gen() -> argparse.ArgumentParser: help='Enable machine-readable output.') readable.add_argument('--machine-readable-json', action='store_true', help='Enable machine-readable output (JSON).') - parser_x.add_argument('templates', nargs='*', metavar='TEMPLATE') + parser_x.add_argument('templates', nargs='*', metavar='TEMPLATESPEC') # qvm-template search parser_search = parser_add_command('search', help_str='Search template details for the given string.') parser_search.add_argument('--all', action='store_true', - help=('Search also in template description and URL. In addition,' + help=('Search also in the template description and URL. In addition,' ' the criterion are evaluated with OR instead of AND.')) parser_search.add_argument('templates', nargs='*', metavar='PATTERN') # qvm-template remove parser_remove = parser_add_command('remove', help_str='Remove installed templates.') parser_remove.add_argument('--disassoc', action='store_true', - help='Also disassociate VMs from the templates to be removed.') + help=('Also disassociate VMs from the templates to be removed.' + ' This creates a dummy template for the VMs to link with.')) parser_remove.add_argument('templates', nargs='*', metavar='TEMPLATE') # qvm-template purge parser_purge = parser_add_command('purge', @@ -153,7 +161,7 @@ def parser_gen() -> argparse.ArgumentParser: parser_purge.add_argument('templates', nargs='*', metavar='TEMPLATE') # qvm-template clean parser_clean = parser_add_command('clean', - help_str='Remove cached data.') + help_str='Remove locally cached packages.') _ = parser_clean # unused # qvm-template repolist parser_repolist = parser_add_command('repolist', @@ -162,9 +170,9 @@ def parser_gen() -> argparse.ArgumentParser: repolim.add_argument('--all', action='store_true', help='Show all repos.') repolim.add_argument('--enabled', action='store_true', - help='Show enabled repos (default).') + help='Show only enabled repos (default).') repolim.add_argument('--disabled', action='store_true', - help='Show disabled repos.') + help='Show only disabled repos.') parser_repolist.add_argument('repos', nargs='*', metavar='REPOS') return parser_main From 2e06e300e6179d200d1bf7a8a2596baa2847fa83 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Wed, 26 Aug 2020 01:31:33 +0800 Subject: [PATCH 056/119] qvm-template: Tweak machine-readable output format. --- doc/manpages/qvm-template.rst | 4 ++-- qubesadmin/tools/qvm_template.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/manpages/qvm-template.rst b/doc/manpages/qvm-template.rst index 2b0c50b..9c1ff00 100644 --- a/doc/manpages/qvm-template.rst +++ b/doc/manpages/qvm-template.rst @@ -219,7 +219,7 @@ Options :: - |{status}|{name}|{evr}|{reponame}| + {status}|{name}|{evr}|{reponame} Where ``{status}`` can be one of ``installed``, ``available``, ``extra``, or ``upgradable``. @@ -307,7 +307,7 @@ Options :: - |{status}|{name}|{epoch}|{version}|{release}|{reponame}|{size}|{buildtime}|{installtime}|{license}|{url}|{summary}|{description}| + {status}|{name}|{epoch}|{version}|{release}|{reponame}|{size}|{buildtime}|{installtime}|{license}|{url}|{summary}|{description} Where ``{status}`` can be one of ``installed``, ``available``, ``extra``, or ``upgradable``. diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 6173623..db04dff 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -1082,9 +1082,8 @@ def list_templates(args: argparse.Namespace, elif operation == 'list': tpl_list = list_to_machine_output(tpl_list) for status, grp in tpl_list.items(): - print('|' + status) for line in grp: - print('|'.join(line.values()) + '|') + print('|'.join([status] + list(line.values()))) elif args.machine_readable_json: if operation == 'info': tpl_list = info_to_machine_output(tpl_list, replace_newline=False) From 32a38c718384ee186d7e5aa098fb423bcd198d92 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 30 Aug 2020 01:58:25 +0800 Subject: [PATCH 057/119] qvm-template: Eliminate use of lsb_release --- qubesadmin/tools/qvm_template.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index db04dff..3c5421c 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -41,8 +41,16 @@ def qubes_release() -> str: with open('/usr/share/qubes/marker-vm', 'r') as fd: # Get last line (in the format `x.x`) return fd.readlines()[-1].strip() - return subprocess.check_output(['lsb_release', '-sr'], - encoding='UTF-8').strip() + with open('/etc/os-release', 'r') as fd: + for line in fd: + line = line.strip() + if not line or line[0] == '#': + continue + key, val = line.split('=', 1) + if key != 'VERSION_ID': + continue + val = val.strip('\'"') # strip possible quotes + return val def parser_gen() -> argparse.ArgumentParser: """Generate argument parser for the application.""" From d65d3c741a301ae3e3733d54b74fc0a33b3d7fe6 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 30 Aug 2020 02:01:19 +0800 Subject: [PATCH 058/119] qvm-template: Replace "template-install-time" with "template-installtime" for consistency --- qubesadmin/tools/qvm_template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 3c5421c..4def912 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -919,7 +919,7 @@ def install( tpl.features['template-buildtime'] = \ str(datetime.datetime.fromtimestamp( int(package_hdr[rpm.RPMTAG_BUILDTIME]))) - tpl.features['template-install-time'] = \ + tpl.features['template-installtime'] = \ str(datetime.datetime.today()) tpl.features['template-license'] = \ package_hdr[rpm.RPMTAG_LICENSE] @@ -1033,7 +1033,7 @@ def list_templates(args: argparse.Namespace, assert False and 'Unknown operation' def append_vm(vm, status): - append(query_local(vm), status, vm.features['template-install-time']) + append(query_local(vm), status, vm.features['template-installtime']) if not (args.installed or args.available or args.extras or args.upgrades): args.all = True From 6b3858314da98bc6aa72c3d1e27c7a9a97ccb299 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 30 Aug 2020 02:54:43 +0800 Subject: [PATCH 059/119] qvm-template: Improve help message for --upgrades --- doc/manpages/qvm-template.rst | 4 ++-- qubesadmin/tools/qvm_template.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manpages/qvm-template.rst b/doc/manpages/qvm-template.rst index 9c1ff00..81d84fb 100644 --- a/doc/manpages/qvm-template.rst +++ b/doc/manpages/qvm-template.rst @@ -208,7 +208,7 @@ Options .. option:: --upgrades - Show upgradable templates. + Show available upgrades. .. option:: --machine-readable @@ -296,7 +296,7 @@ Options .. option:: --upgrades - Show upgradable templates. + Show available upgrades. .. option:: --machine-readable diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 4def912..7457924 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -142,7 +142,7 @@ def parser_gen() -> argparse.ArgumentParser: help=('Show extras (e.g., ones that exist' ' locally but not in repos) templates.')) parser_x.add_argument('--upgrades', action='store_true', - help='Show upgradable templates.') + help='Show available upgrades.') readable = parser_x.add_mutually_exclusive_group() readable.add_argument('--machine-readable', action='store_true', help='Enable machine-readable output.') From 4199a9a222886e1b4769595fa97f30b2d32d90cb Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Mon, 31 Aug 2020 01:48:31 +0800 Subject: [PATCH 060/119] qvm-template: Fix qvm_template_postprocess tests --- .../tests/tools/qvm_template_postprocess.py | 78 ++++++++++++++++--- 1 file changed, 67 insertions(+), 11 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template_postprocess.py b/qubesadmin/tests/tools/qvm_template_postprocess.py index 1a26d8e..de601f8 100644 --- a/qubesadmin/tests/tools/qvm_template_postprocess.py +++ b/qubesadmin/tests/tools/qvm_template_postprocess.py @@ -176,18 +176,46 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase): self.assertAllCalled() def test_010_import_appmenus(self): + default_menu_items = [ + 'org.gnome.Terminal.desktop', + 'firefox.desktop'] + menu_items = [ + 'org.gnome.Terminal.desktop', + 'org.gnome.Software.desktop', + 'gnome-control-center.desktop'] + netvm_menu_items = [ + 'org.gnome.Terminal.desktop', + 'nm-connection-editor.desktop'] with open(os.path.join(self.source_dir.name, 'vm-whitelisted-appmenus.list'), 'w') as f: - f.write('org.gnome.Terminal.desktop\n') - f.write('firefox.desktop\n') + for entry in default_menu_items: + f.write(entry + '\n') with open(os.path.join(self.source_dir.name, 'whitelisted-appmenus.list'), 'w') as f: - f.write('org.gnome.Terminal.desktop\n') - f.write('org.gnome.Software.desktop\n') - f.write('gnome-control-center.desktop\n') + for entry in menu_items: + f.write(entry + '\n') + with open(os.path.join(self.source_dir.name, + 'netvm-whitelisted-appmenus.list'), 'w') as f: + for entry in netvm_menu_items: + f.write(entry + '\n') self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ b'0\0test-vm class=TemplateVM state=Halted\n' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Set', + 'default-menu-items', + ' '.join(default_menu_items).encode())] = b'0\0' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Set', + 'menu-items', + ' '.join(menu_items).encode())] = b'0\0' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Set', + 'netvm-menu-items', + ' '.join(netvm_menu_items).encode())] = b'0\0' vm = self.app.domains['test-vm'] with mock.patch('subprocess.check_call') as mock_proc: @@ -205,15 +233,43 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase): @mock.patch('grp.getgrnam') @mock.patch('os.getuid') def test_011_import_appmenus_as_root(self, mock_getuid, mock_getgrnam): + default_menu_items = [ + 'org.gnome.Terminal.desktop', + 'firefox.desktop'] + menu_items = [ + 'org.gnome.Terminal.desktop', + 'org.gnome.Software.desktop', + 'gnome-control-center.desktop'] + netvm_menu_items = [ + 'org.gnome.Terminal.desktop', + 'nm-connection-editor.desktop'] with open(os.path.join(self.source_dir.name, 'vm-whitelisted-appmenus.list'), 'w') as f: - f.write('org.gnome.Terminal.desktop\n') - f.write('firefox.desktop\n') + for entry in default_menu_items: + f.write(entry + '\n') with open(os.path.join(self.source_dir.name, 'whitelisted-appmenus.list'), 'w') as f: - f.write('org.gnome.Terminal.desktop\n') - f.write('org.gnome.Software.desktop\n') - f.write('gnome-control-center.desktop\n') + for entry in menu_items: + f.write(entry + '\n') + with open(os.path.join(self.source_dir.name, + 'netvm-whitelisted-appmenus.list'), 'w') as f: + for entry in netvm_menu_items: + f.write(entry + '\n') + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Set', + 'default-menu-items', + ' '.join(default_menu_items).encode())] = b'0\0' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Set', + 'menu-items', + ' '.join(menu_items).encode())] = b'0\0' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Set', + 'netvm-menu-items', + ' '.join(netvm_menu_items).encode())] = b'0\0' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ b'0\0test-vm class=TemplateVM state=Halted\n' @@ -313,7 +369,7 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase): app=self.app) self.assertEqual(ret, 0) self.app.add_new_vm.assert_called_once_with('TemplateVM', - name='test-vm', label='black') + name='test-vm', label='black', pool=None) mock_import_root_img.assert_called_once_with(self.app.domains[ 'test-vm'], self.source_dir.name) mock_import_appmenus.assert_called_once_with(self.app.domains[ From 39492ffce930d935ae58f257e92121070a6fddaa Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Mon, 31 Aug 2020 02:02:12 +0800 Subject: [PATCH 061/119] Fix CI dependencies --- ci/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/requirements.txt b/ci/requirements.txt index d89e944..b216357 100644 --- a/ci/requirements.txt +++ b/ci/requirements.txt @@ -10,3 +10,5 @@ lxml PyYAML xcffib asynctest +tqdm +pyxdg From 3a42564af22fdb2f828c08f951a0891ed3963aa5 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Mon, 31 Aug 2020 02:08:44 +0800 Subject: [PATCH 062/119] qvm-template: Make pylint happy --- .pylintrc | 2 ++ qubesadmin/tools/qvm_template.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.pylintrc b/.pylintrc index 689b0e6..64286eb 100644 --- a/.pylintrc +++ b/.pylintrc @@ -148,6 +148,8 @@ ext-import-graph= # not be disabled) int-import-graph= +ignored-modules=rpm,dnf + [DESIGN] diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 7457924..2f8b5d3 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -20,13 +20,14 @@ import tempfile import time import typing +import rpm +import tqdm +import xdg.BaseDirectory + import qubesadmin import qubesadmin.tools import qubesadmin.tools.qvm_kill import qubesadmin.tools.qvm_remove -import rpm -import tqdm -import xdg.BaseDirectory PATH_PREFIX = '/var/lib/qubes/vm-templates' TEMP_DIR = '/var/tmp' From 757bb333296e19c5d2bf1e091f4a1648ff4bc9c4 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Fri, 4 Sep 2020 01:56:15 +0800 Subject: [PATCH 063/119] Add stubs for rpm module and initial tests for qvm-template install --- .pylintrc | 2 +- qubesadmin/tests/tools/qvm_template.py | 771 +++++++++++++++++++++++++ qubesadmin/tools/qvm_template.py | 5 +- test-packages/rpm.py | 44 ++ 4 files changed, 820 insertions(+), 2 deletions(-) create mode 100644 qubesadmin/tests/tools/qvm_template.py create mode 100644 test-packages/rpm.py diff --git a/.pylintrc b/.pylintrc index 64286eb..0f926da 100644 --- a/.pylintrc +++ b/.pylintrc @@ -148,7 +148,7 @@ ext-import-graph= # not be disabled) int-import-graph= -ignored-modules=rpm,dnf +ignored-modules=dnf [DESIGN] diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py new file mode 100644 index 0000000..dbbc687 --- /dev/null +++ b/qubesadmin/tests/tools/qvm_template.py @@ -0,0 +1,771 @@ +from unittest import mock +import argparse +import asyncio +import datetime +import io +import os +import pathlib +import subprocess +import tempfile + +import rpm + +import qubesadmin.tests +import qubesadmin.tools.qvm_template + +class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): + def setUp(self): + super().setUp() + + def tearDown(self): + super().tearDown() + + def test_000_verify_rpm_success(self): + ts = mock.MagicMock() + # Just return a dict instead of rpm.hdr + hdr = { + rpm.RPMTAG_SIGPGP: 'xxx', # non-empty + rpm.RPMTAG_SIGGPG: 'xxx', # non-empty + } + ts.hdrFromFdno.return_value = hdr + ret = qubesadmin.tools.qvm_template.verify_rpm('/dev/null', ts) + ts.hdrFromFdno.assert_called_once() + self.assertEqual(hdr, ret) + self.assertAllCalled() + + def test_001_verify_rpm_nosig_fail(self): + ts = mock.MagicMock() + # Just return a dict instead of rpm.hdr + hdr = { + rpm.RPMTAG_SIGPGP: None, # empty + rpm.RPMTAG_SIGGPG: None, # empty + } + ts.hdrFromFdno.return_value = hdr + ret = qubesadmin.tools.qvm_template.verify_rpm('/dev/null', ts) + ts.hdrFromFdno.assert_called_once() + self.assertEqual(ret, None) + self.assertAllCalled() + + def test_002_verify_rpm_nosig_success(self): + ts = mock.MagicMock() + # Just return a dict instead of rpm.hdr + hdr = { + rpm.RPMTAG_SIGPGP: None, # empty + rpm.RPMTAG_SIGGPG: None, # empty + } + ts.hdrFromFdno.return_value = hdr + ret = qubesadmin.tools.qvm_template.verify_rpm('/dev/null', ts, True) + ts.hdrFromFdno.assert_called_once() + self.assertEqual(ret, hdr) + self.assertAllCalled() + + def test_003_verify_rpm_badsig_fail(self): + ts = mock.MagicMock() + def f(*args): + raise rpm.error('public key not trusted') + ts.hdrFromFdno.side_effect = f + ret = qubesadmin.tools.qvm_template.verify_rpm('/dev/null', ts) + ts.hdrFromFdno.assert_called_once() + self.assertEqual(ret, None) + self.assertAllCalled() + + @mock.patch('subprocess.Popen') + def test_010_extract_rpm_success(self, mock_popen): + pipe = mock.Mock() + mock_popen.return_value.stdout = pipe + mock_popen.return_value.wait.return_value = 0 + with tempfile.NamedTemporaryFile() as fd, \ + tempfile.TemporaryDirectory() as dir: + path = fd.name + dirpath = dir + ret = qubesadmin.tools.qvm_template.extract_rpm( + 'test-vm', path, dirpath) + self.assertEqual(ret, True) + self.assertEqual(mock_popen.mock_calls, [ + mock.call(['rpm2cpio', path], stdout=subprocess.PIPE), + mock.call([ + 'cpio', + '-idm', + '-D', + dirpath, + './var/lib/qubes/vm-templates/test-vm/*' + ], stdin=pipe, stdout=subprocess.DEVNULL), + mock.call().wait(), + mock.call().wait() + ]) + self.assertAllCalled() + + @mock.patch('subprocess.Popen') + def test_011_extract_rpm_fail(self, mock_popen): + pipe = mock.Mock() + mock_popen.return_value.stdout = pipe + mock_popen.return_value.wait.return_value = 1 + with tempfile.NamedTemporaryFile() as fd, \ + tempfile.TemporaryDirectory() as dir: + path = fd.name + dirpath = dir + ret = qubesadmin.tools.qvm_template.extract_rpm( + 'test-vm', path, dirpath) + self.assertEqual(ret, False) + self.assertEqual(mock_popen.mock_calls, [ + mock.call(['rpm2cpio', path], stdout=subprocess.PIPE), + mock.call([ + 'cpio', + '-idm', + '-D', + dirpath, + './var/lib/qubes/vm-templates/test-vm/*' + ], stdin=pipe, stdout=subprocess.DEVNULL), + mock.call().wait() + ]) + self.assertAllCalled() + + def add_new_vm_side_effect(self, *args, **kwargs): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\0test-vm class=TemplateVM state=Halted\n' + self.app.domains.clear_cache() + return self.app.domains['test-vm'] + + @mock.patch('os.remove') + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') + def test_100_install_local_success( + self, + mock_ts, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename, + mock_remove): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = b'0\0' + build_time = '2020-09-01 22:30:00' # 1598970600 + install_time = '2020-09-01 23:30:00.508230' + for key, val in [ + ('name', 'test-vm'), + ('epoch', '2'), + ('version', '4.1'), + ('release', '2020'), + ('reponame', '@commandline'), + ('buildtime', build_time), + ('installtime', install_time), + ('license', 'GPL'), + ('url', 'https://qubes-os.org'), + ('summary', 'Summary'), + ('description', 'Desc|desc')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Set', + f'template-{key}', + val.encode())] = b'0\0' + mock_verify.return_value = { + rpm.RPMTAG_NAME : 'qubes-template-test-vm', + rpm.RPMTAG_BUILDTIME : 1598970600, + rpm.RPMTAG_DESCRIPTION : 'Desc\ndesc', + rpm.RPMTAG_EPOCHNUM : 2, + rpm.RPMTAG_LICENSE : 'GPL', + rpm.RPMTAG_RELEASE : '2020', + rpm.RPMTAG_SUMMARY : 'Summary', + rpm.RPMTAG_URL : 'https://qubes-os.org', + rpm.RPMTAG_VERSION : '4.1' + } + mock_dl_list.return_value = {} + mock_call.side_effect = self.add_new_vm_side_effect + mock_time = mock.Mock(wraps=datetime.datetime) + mock_time.today.return_value = \ + datetime.datetime.fromisoformat(install_time) + with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + mock.patch('datetime.datetime', new=mock_time), \ + mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ + tempfile.NamedTemporaryFile(suffix='.rpm') as template_file: + path = template_file.name + args = argparse.Namespace( + templates=[path], + keyring='/usr/share/qubes/repo-templates/keys', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + yes=False, + allow_pv=False, + pool=None + ) + mock_tmpdir.return_value.__enter__.return_value = \ + '/var/tmp/qvm-template-tmpdir' + qubesadmin.tools.qvm_template.install(args, self.app) + # Lock file created + self.assertEqual(mock_open.mock_calls, [ + mock.call('/var/tmp/qvm-template.lck', 'x'), + mock.call().__enter__(), + mock.call().__exit__(None, None, None) + ]) + # Keyring created + self.assertEqual(mock_ts.mock_calls, [ + mock.call('/usr/share/qubes/repo-templates/keys') + ]) + # Package verified + self.assertEqual(mock_verify.mock_calls, [ + mock.call(path, mock_ts('/usr/share/qubes/repo-templates/keys'), + False) + ]) + # Attempt to get download list + selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST + self.assertEqual(mock_dl_list.mock_calls, [ + mock.call(args, self.app, version_selector=selector) + ]) + # Nothing downloaded + self.assertEqual(mock_dl.mock_calls, [ + mock.call(args, self.app, path_override='/var/cache/qvm-template', + dl_list={}, suffix='.unverified', version_selector=selector) + ]) + # Package is extracted + self.assertEqual(mock_extract.mock_calls, [ + mock.call('test-vm', path, '/var/tmp/qvm-template-tmpdir') + ]) + # No packages overwritten, so no confirm needed + self.assertEqual(mock_confirm.mock_calls, []) + # qvm-template-postprocess is called + self.assertEqual(mock_call.mock_calls, [ + mock.call([ + 'qvm-template-postprocess', + '--really', + '--no-installed-by-rpm', + 'post-install', + 'test-vm', + '/var/tmp/qvm-template-tmpdir' + '/var/lib/qubes/vm-templates/test-vm' + ]) + ]) + # Cache directory created + self.assertEqual(mock_mkdirs.mock_calls, [ + mock.call(args.cachedir, exist_ok=True) + ]) + # No templates downloaded, thus no renames needed + self.assertEqual(mock_rename.mock_calls, []) + # Lock file removed + self.assertEqual(mock_remove.mock_calls, [ + mock.call('/var/tmp/qvm-template.lck') + ]) + self.assertAllCalled() + + @mock.patch('os.remove') + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') + def test_101_install_local_postprocargs_success( + self, + mock_ts, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename, + mock_remove): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = b'0\0' + build_time = '2020-09-01 22:30:00' # 1598970600 + install_time = '2020-09-01 23:30:00.508230' + for key, val in [ + ('name', 'test-vm'), + ('epoch', '2'), + ('version', '4.1'), + ('release', '2020'), + ('reponame', '@commandline'), + ('buildtime', build_time), + ('installtime', install_time), + ('license', 'GPL'), + ('url', 'https://qubes-os.org'), + ('summary', 'Summary'), + ('description', 'Desc|desc')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Set', + f'template-{key}', + val.encode())] = b'0\0' + mock_verify.return_value = { + rpm.RPMTAG_NAME : 'qubes-template-test-vm', + rpm.RPMTAG_BUILDTIME : 1598970600, + rpm.RPMTAG_DESCRIPTION : 'Desc\ndesc', + rpm.RPMTAG_EPOCHNUM : 2, + rpm.RPMTAG_LICENSE : 'GPL', + rpm.RPMTAG_RELEASE : '2020', + rpm.RPMTAG_SUMMARY : 'Summary', + rpm.RPMTAG_URL : 'https://qubes-os.org', + rpm.RPMTAG_VERSION : '4.1' + } + mock_dl_list.return_value = {} + mock_call.side_effect = self.add_new_vm_side_effect + mock_time = mock.Mock(wraps=datetime.datetime) + mock_time.today.return_value = \ + datetime.datetime.fromisoformat(install_time) + with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + mock.patch('datetime.datetime', new=mock_time), \ + mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ + tempfile.NamedTemporaryFile(suffix='.rpm') as template_file: + path = template_file.name + args = argparse.Namespace( + templates=[path], + keyring='/usr/share/qubes/repo-templates/keys', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + yes=False, + allow_pv=True, + pool='my-pool' + ) + mock_tmpdir.return_value.__enter__.return_value = \ + '/var/tmp/qvm-template-tmpdir' + qubesadmin.tools.qvm_template.install(args, self.app) + # Lock file created + self.assertEqual(mock_open.mock_calls, [ + mock.call('/var/tmp/qvm-template.lck', 'x'), + mock.call().__enter__(), + mock.call().__exit__(None, None, None) + ]) + # Keyring created + self.assertEqual(mock_ts.mock_calls, [ + mock.call('/usr/share/qubes/repo-templates/keys') + ]) + # Package verified + self.assertEqual(mock_verify.mock_calls, [ + mock.call(path, mock_ts('/usr/share/qubes/repo-templates/keys'), + False) + ]) + # Attempt to get download list + selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST + self.assertEqual(mock_dl_list.mock_calls, [ + mock.call(args, self.app, version_selector=selector) + ]) + # Nothing downloaded + self.assertEqual(mock_dl.mock_calls, [ + mock.call(args, self.app, path_override='/var/cache/qvm-template', + dl_list={}, suffix='.unverified', version_selector=selector) + ]) + # Package is extracted + self.assertEqual(mock_extract.mock_calls, [ + mock.call('test-vm', path, '/var/tmp/qvm-template-tmpdir') + ]) + # No packages overwritten, so no confirm needed + self.assertEqual(mock_confirm.mock_calls, []) + # qvm-template-postprocess is called + self.assertEqual(mock_call.mock_calls, [ + mock.call([ + 'qvm-template-postprocess', + '--really', + '--no-installed-by-rpm', + '--allow-pv', + '--pool', + 'my-pool', + 'post-install', + 'test-vm', + '/var/tmp/qvm-template-tmpdir' + '/var/lib/qubes/vm-templates/test-vm' + ]) + ]) + # Cache directory created + self.assertEqual(mock_mkdirs.mock_calls, [ + mock.call(args.cachedir, exist_ok=True) + ]) + # No templates downloaded, thus no renames needed + self.assertEqual(mock_rename.mock_calls, []) + # Lock file removed + self.assertEqual(mock_remove.mock_calls, [ + mock.call('/var/tmp/qvm-template.lck') + ]) + self.assertAllCalled() + + @mock.patch('os.remove') + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') + def test_102_install_local_badsig_fail( + self, + mock_ts, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename, + mock_remove): + mock_verify.return_value = None + mock_time = mock.Mock(wraps=datetime.datetime) + with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + mock.patch('datetime.datetime', new=mock_time), \ + mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ + tempfile.NamedTemporaryFile(suffix='.rpm') as template_file: + path = template_file.name + args = argparse.Namespace( + templates=[path], + keyring='/usr/share/qubes/repo-templates/keys', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + yes=False, + allow_pv=False, + pool=None + ) + mock_tmpdir.return_value.__enter__.return_value = \ + '/var/tmp/qvm-template-tmpdir' + # Should raise parser.error + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.install(args, self.app) + # Lock file created + self.assertEqual(mock_open.mock_calls, [ + mock.call('/var/tmp/qvm-template.lck', 'x'), + mock.call().__enter__(), + mock.call().__exit__(None, None, None) + ]) + # Check error message + self.assertTrue('verification failed' in mock_err.getvalue()) + # Keyring created + self.assertEqual(mock_ts.mock_calls, [ + mock.call('/usr/share/qubes/repo-templates/keys') + ]) + # Package verified + self.assertEqual(mock_verify.mock_calls, [ + mock.call(path, mock_ts('/usr/share/qubes/repo-templates/keys'), + False) + ]) + # Should not be executed: + self.assertEqual(mock_dl_list.mock_calls, []) + self.assertEqual(mock_dl.mock_calls, []) + self.assertEqual(mock_extract.mock_calls, []) + self.assertEqual(mock_confirm.mock_calls, []) + self.assertEqual(mock_call.mock_calls, []) + self.assertEqual(mock_rename.mock_calls, []) + # Lock file removed + self.assertEqual(mock_remove.mock_calls, [ + mock.call('/var/tmp/qvm-template.lck') + ]) + self.assertAllCalled() + + @mock.patch('os.remove') + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') + def test_103_install_local_exists_fail( + self, + mock_ts, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename, + mock_remove): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\0test-vm class=TemplateVM state=Halted\n' + mock_verify.return_value = { + rpm.RPMTAG_NAME : 'qubes-template-test-vm', + rpm.RPMTAG_BUILDTIME : 1598970600, + rpm.RPMTAG_DESCRIPTION : 'Desc\ndesc', + rpm.RPMTAG_EPOCHNUM : 2, + rpm.RPMTAG_LICENSE : 'GPL', + rpm.RPMTAG_RELEASE : '2020', + rpm.RPMTAG_SUMMARY : 'Summary', + rpm.RPMTAG_URL : 'https://qubes-os.org', + rpm.RPMTAG_VERSION : '4.1' + } + mock_dl_list.return_value = {} + mock_time = mock.Mock(wraps=datetime.datetime) + with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + mock.patch('datetime.datetime', new=mock_time), \ + mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ + tempfile.NamedTemporaryFile(suffix='.rpm') as template_file: + path = template_file.name + args = argparse.Namespace( + templates=[path], + keyring='/usr/share/qubes/repo-templates/keys', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + yes=False, + allow_pv=False, + pool=None + ) + mock_tmpdir.return_value.__enter__.return_value = \ + '/var/tmp/qvm-template-tmpdir' + qubesadmin.tools.qvm_template.install(args, self.app) + # Lock file created + self.assertEqual(mock_open.mock_calls, [ + mock.call('/var/tmp/qvm-template.lck', 'x'), + mock.call().__enter__(), + mock.call().__exit__(None, None, None) + ]) + # Check warning message + self.assertTrue('already installed' in mock_err.getvalue()) + # Keyring created + self.assertEqual(mock_ts.mock_calls, [ + mock.call('/usr/share/qubes/repo-templates/keys') + ]) + # Package verified + self.assertEqual(mock_verify.mock_calls, [ + mock.call(path, mock_ts('/usr/share/qubes/repo-templates/keys'), + False) + ]) + # Attempt to get download list + selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST + self.assertEqual(mock_dl_list.mock_calls, [ + mock.call(args, self.app, version_selector=selector) + ]) + # Nothing downloaded + self.assertEqual(mock_dl.mock_calls, [ + mock.call(args, self.app, path_override='/var/cache/qvm-template', + dl_list={}, suffix='.unverified', version_selector=selector) + ]) + # Should not be executed: + self.assertEqual(mock_extract.mock_calls, []) + self.assertEqual(mock_confirm.mock_calls, []) + self.assertEqual(mock_call.mock_calls, []) + self.assertEqual(mock_rename.mock_calls, []) + # Lock file removed + self.assertEqual(mock_remove.mock_calls, [ + mock.call('/var/tmp/qvm-template.lck') + ]) + self.assertAllCalled() + + @mock.patch('os.remove') + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') + def test_104_install_local_badpkgname_fail( + self, + mock_ts, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename, + mock_remove): + mock_verify.return_value = { + rpm.RPMTAG_NAME : 'Xqubes-template-test-vm', + rpm.RPMTAG_BUILDTIME : 1598970600, + rpm.RPMTAG_DESCRIPTION : 'Desc\ndesc', + rpm.RPMTAG_EPOCHNUM : 2, + rpm.RPMTAG_LICENSE : 'GPL', + rpm.RPMTAG_RELEASE : '2020', + rpm.RPMTAG_SUMMARY : 'Summary', + rpm.RPMTAG_URL : 'https://qubes-os.org', + rpm.RPMTAG_VERSION : '4.1' + } + mock_time = mock.Mock(wraps=datetime.datetime) + with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + mock.patch('datetime.datetime', new=mock_time), \ + mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ + tempfile.NamedTemporaryFile(suffix='.rpm') as template_file: + path = template_file.name + args = argparse.Namespace( + templates=[path], + keyring='/usr/share/qubes/repo-templates/keys', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + yes=False, + allow_pv=False, + pool=None + ) + mock_tmpdir.return_value.__enter__.return_value = \ + '/var/tmp/qvm-template-tmpdir' + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.install(args, self.app) + # Lock file created + self.assertEqual(mock_open.mock_calls, [ + mock.call('/var/tmp/qvm-template.lck', 'x'), + mock.call().__enter__(), + mock.call().__exit__(None, None, None) + ]) + # Check error message + self.assertTrue('Illegal package name' in mock_err.getvalue()) + # Keyring created + self.assertEqual(mock_ts.mock_calls, [ + mock.call('/usr/share/qubes/repo-templates/keys') + ]) + # Package verified + self.assertEqual(mock_verify.mock_calls, [ + mock.call(path, mock_ts('/usr/share/qubes/repo-templates/keys'), + False) + ]) + # Should not be executed: + self.assertEqual(mock_dl_list.mock_calls, []) + self.assertEqual(mock_dl.mock_calls, []) + self.assertEqual(mock_extract.mock_calls, []) + self.assertEqual(mock_confirm.mock_calls, []) + self.assertEqual(mock_call.mock_calls, []) + self.assertEqual(mock_rename.mock_calls, []) + # Lock file removed + self.assertEqual(mock_remove.mock_calls, [ + mock.call('/var/tmp/qvm-template.lck') + ]) + self.assertAllCalled() + + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') + def test_105_install_local_existinginstance_fail( + self, + mock_ts, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename): + mock_time = mock.Mock(wraps=datetime.datetime) + with mock.patch('datetime.datetime', new=mock_time), \ + mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ + tempfile.NamedTemporaryFile(suffix='.rpm') as template_file: + path = template_file.name + args = argparse.Namespace( + templates=[path], + keyring='/usr/share/qubes/repo-templates/keys', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + yes=False, + allow_pv=False, + pool=None + ) + mock_tmpdir.return_value.__enter__.return_value = \ + '/var/tmp/qvm-template-tmpdir' + pathlib.Path('/var/tmp/qvm-template.lck').touch() + try: + with self.assertRaises(SystemExit), \ + mock.patch('os.remove') as mock_remove: + qubesadmin.tools.qvm_template.install(args, self.app) + self.assertEqual(mock_remove.mock_calls, []) + finally: + # Lock file not removed + self.assertTrue(os.path.exists('/var/tmp/qvm-template.lck')) + os.remove('/var/tmp/qvm-template.lck') + # Check error message + self.assertTrue('another instance of qvm-template is running' \ + in mock_err.getvalue()) + # Should not be executed: + self.assertEqual(mock_ts.mock_calls, []) + self.assertEqual(mock_verify.mock_calls, []) + self.assertEqual(mock_dl_list.mock_calls, []) + self.assertEqual(mock_dl.mock_calls, []) + self.assertEqual(mock_extract.mock_calls, []) + self.assertEqual(mock_confirm.mock_calls, []) + self.assertEqual(mock_call.mock_calls, []) + self.assertEqual(mock_rename.mock_calls, []) + self.assertAllCalled() + + @mock.patch('os.remove') + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') + def test_106_install_local_badpath_fail( + self, + mock_ts, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename, + mock_remove): + mock_time = mock.Mock(wraps=datetime.datetime) + with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + mock.patch('datetime.datetime', new=mock_time), \ + mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + path = '/var/tmp/ShOulD-NoT-ExIsT.rpm' + args = argparse.Namespace( + templates=[path], + keyring='/usr/share/qubes/repo-templates/keys', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + yes=False, + allow_pv=False, + pool=None + ) + mock_tmpdir.return_value.__enter__.return_value = \ + '/var/tmp/qvm-template-tmpdir' + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.install(args, self.app) + # Lock file created + self.assertEqual(mock_open.mock_calls, [ + mock.call('/var/tmp/qvm-template.lck', 'x'), + mock.call().__enter__(), + mock.call().__exit__(None, None, None) + ]) + # Check error message + self.assertTrue(f"RPM file '{path}' not found" \ + in mock_err.getvalue()) + # Keyring created + self.assertEqual(mock_ts.mock_calls, [ + mock.call('/usr/share/qubes/repo-templates/keys') + ]) + # Should not be executed: + self.assertEqual(mock_verify.mock_calls, []) + self.assertEqual(mock_dl_list.mock_calls, []) + self.assertEqual(mock_dl.mock_calls, []) + self.assertEqual(mock_extract.mock_calls, []) + self.assertEqual(mock_confirm.mock_calls, []) + self.assertEqual(mock_call.mock_calls, []) + self.assertEqual(mock_rename.mock_calls, []) + # Lock file removed + self.assertEqual(mock_remove.mock_calls, [ + mock.call('/var/tmp/qvm-template.lck') + ]) + self.assertAllCalled() diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 2f8b5d3..301c8f0 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -286,7 +286,7 @@ def query_local(vm: qubesadmin.vm.QubesVM) -> Template: vm.features['template-release'], vm.features['template-reponame'], vm.get_disk_utilization(), - vm.features['template-buildtime'], + datetime.datetime.fromisoformat(vm.features['template-buildtime']), vm.features['template-license'], vm.features['template-url'], vm.features['template-summary'], @@ -572,6 +572,8 @@ def verify_rpm( except rpm.error as e: if str(e) == 'public key not trusted' \ or str(e) == 'public key not available': + # TODO: This does not work + # Should just tell TransactionSet not to verify sigs return hdr if nogpgcheck else None return None return hdr @@ -891,6 +893,7 @@ def install( for rpmfile, reponame, name, package_hdr in verified_rpm_list: with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: print('Installing template \'%s\'...' % name, file=sys.stderr) + # TODO: Handle return value extract_rpm(name, rpmfile, target) cmdline = [ 'qvm-template-postprocess', diff --git a/test-packages/rpm.py b/test-packages/rpm.py new file mode 100644 index 0000000..c167adb --- /dev/null +++ b/test-packages/rpm.py @@ -0,0 +1,44 @@ +# RPM header tags +# Generated with the following command: +# ``grep -Po '(RPMTAG[A-Z_]*)' tools/qvm_template.py | sort | uniq`` +RPMTAG_BUILDTIME = 1 +RPMTAG_DESCRIPTION = 2 +RPMTAG_EPOCHNUM = 3 +RPMTAG_LICENSE = 4 +RPMTAG_NAME = 5 +RPMTAG_RELEASE = 6 +RPMTAG_SIGGPG = 7 +RPMTAG_SIGPGP = 8 +RPMTAG_SUMMARY = 9 +RPMTAG_URL = 10 +RPMTAG_VERSION = 11 + +class error(BaseException): + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return self.msg + +class hdr(): + pass + +class keyring(): + def addKey(self, *args): + pass + +class pubkey(): + pass + +class TransactionSet(): + def setKeyring(self, *args): + pass + +class transaction(): + class TransactionSet(): + def setKeyring(self, *args): + pass + +def labelCompare(a, b): + # Pretend that we're comparing the versions lexographically in the stub + return (a > b) - (a < b) From 5a1e1b7fdd049a648cd178dda6d056b2cc8a1d2e Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Fri, 4 Sep 2020 01:59:28 +0800 Subject: [PATCH 064/119] qvm-template: Update docs for --updatevm --- doc/manpages/qvm-template.rst | 3 ++- qubesadmin/tools/qvm_template.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/manpages/qvm-template.rst b/doc/manpages/qvm-template.rst index 81d84fb..cbc8d25 100644 --- a/doc/manpages/qvm-template.rst +++ b/doc/manpages/qvm-template.rst @@ -30,7 +30,8 @@ Options .. option:: --updatevm UPDATEVM - Specify VM to download updates from. (default: sys-firewall) + Specify VM to download updates from. (Set to empty string to specify the + current VM.) (default: sys-firewall) .. option:: --enablerepo REPOID diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 301c8f0..6e43282 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -76,7 +76,8 @@ def parser_gen() -> argparse.ArgumentParser: default='/usr/share/qubes/repo-templates/keys', help='Specify directory containing RPM public keys.') parser_main.add_argument('--updatevm', default='sys-firewall', - help='Specify VM to download updates from.') + help=('Specify VM to download updates from.' + ' (Set to empty string to specify the current VM.)')) parser_main.add_argument('--enablerepo', action='append', default=[], metavar='REPOID', help=('Enable additional repositories by an id or a glob.' @@ -340,7 +341,7 @@ def qrexec_popen( ``args.updatevm``. Note that this falls back to invoking ``/etc/qubes-rpc/*`` directly if - ``args.updatevm`` is None. + ``args.updatevm`` is empty string. :param args: Arguments received by the application. ``args.updatevm`` is used From 205eee4d805079bfe46346d501f4dd0c80a6016a Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 5 Sep 2020 01:51:45 +0800 Subject: [PATCH 065/119] qvm-template: Fix timezone issues by storing timezone explictly in features --- qubesadmin/tests/tools/qvm_template.py | 8 ++++---- qubesadmin/tools/qvm_template.py | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index dbbc687..99691a5 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -149,8 +149,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_rename, mock_remove): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = b'0\0' - build_time = '2020-09-01 22:30:00' # 1598970600 - install_time = '2020-09-01 23:30:00.508230' + build_time = '2020-09-01 14:30:00+00:00' # 1598970600 + install_time = '2020-09-01 15:30:00.508230+00:00' for key, val in [ ('name', 'test-vm'), ('epoch', '2'), @@ -280,8 +280,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_rename, mock_remove): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = b'0\0' - build_time = '2020-09-01 22:30:00' # 1598970600 - install_time = '2020-09-01 23:30:00.508230' + build_time = '2020-09-01 14:30:00+00:00' # 1598970600 + install_time = '2020-09-01 15:30:00.508230+00:00' for key, val in [ ('name', 'test-vm'), ('epoch', '2'), diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 6e43282..fcc3fdc 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -923,9 +923,10 @@ def install( tpl.features['template-reponame'] = reponame tpl.features['template-buildtime'] = \ str(datetime.datetime.fromtimestamp( - int(package_hdr[rpm.RPMTAG_BUILDTIME]))) + int(package_hdr[rpm.RPMTAG_BUILDTIME]), + tz=datetime.timezone.utc)) tpl.features['template-installtime'] = \ - str(datetime.datetime.today()) + str(datetime.datetime.today(tz=datetime.timezone.utc)) tpl.features['template-license'] = \ package_hdr[rpm.RPMTAG_LICENSE] tpl.features['template-url'] = \ From 199996e7b8757b6872b5064d39527fd3491d8e94 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 5 Sep 2020 02:06:53 +0800 Subject: [PATCH 066/119] qvm-template: Fix compatibility with Python 3.6 --- qubesadmin/tools/qvm_template.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index fcc3fdc..17f6530 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -58,7 +58,7 @@ def parser_gen() -> argparse.ArgumentParser: formatter = argparse.ArgumentDefaultsHelpFormatter parser_main = argparse.ArgumentParser(description='Qubes Template Manager', formatter_class=formatter) - subparsers = parser_main.add_subparsers(dest='operation', required=True, + subparsers = parser_main.add_subparsers(dest='operation', description='Command to run.') def parser_add_command(cmd, help_str): @@ -1363,6 +1363,9 @@ def main(args: typing.Optional[typing.Sequence[str]] = None, """ p_args = parser.parse_args(args) + if not p_args.operation: + parser.error('An operation needs to be specified.') + # If the user specified other repo files... if len(p_args.repo_files) > 1: # ...remove the default entry From f8032b0f5a212cf23e0694a637451abb36badf6b Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 5 Sep 2020 02:25:20 +0800 Subject: [PATCH 067/119] Revert "qvm-template: Fix compatibility with Python 3.6" This reverts commit 199996e7b8757b6872b5064d39527fd3491d8e94. --- qubesadmin/tools/qvm_template.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 17f6530..fcc3fdc 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -58,7 +58,7 @@ def parser_gen() -> argparse.ArgumentParser: formatter = argparse.ArgumentDefaultsHelpFormatter parser_main = argparse.ArgumentParser(description='Qubes Template Manager', formatter_class=formatter) - subparsers = parser_main.add_subparsers(dest='operation', + subparsers = parser_main.add_subparsers(dest='operation', required=True, description='Command to run.') def parser_add_command(cmd, help_str): @@ -1363,9 +1363,6 @@ def main(args: typing.Optional[typing.Sequence[str]] = None, """ p_args = parser.parse_args(args) - if not p_args.operation: - parser.error('An operation needs to be specified.') - # If the user specified other repo files... if len(p_args.repo_files) > 1: # ...remove the default entry From 89895038b5c301c5eefa4a2b5ed83e7df534b74d Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 6 Sep 2020 23:57:42 +0800 Subject: [PATCH 068/119] qvm-template: Fix date formats to "%Y-%m-%d %H:%M:%S" --- doc/manpages/qvm-template.rst | 15 ++++++--------- qubesadmin/tests/tools/qvm_template.py | 8 ++++---- qubesadmin/tools/qvm_template.py | 17 ++++++++++------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/doc/manpages/qvm-template.rst b/doc/manpages/qvm-template.rst index cbc8d25..406f237 100644 --- a/doc/manpages/qvm-template.rst +++ b/doc/manpages/qvm-template.rst @@ -252,9 +252,8 @@ Options Where ``STATUS`` can be one of ``"installed"``, ``"available"``, ``"extra"``, or ``"upgradable"``. - The fields ``buildtime`` and ``installtime`` are in ISO 8601 format. - For example, one can parse them in Python with - ``datetime.fromisoformat()``. + The fields ``buildtime`` and ``installtime`` are in ``%Y-%m-%d + %H:%M:%S`` format in UTC. The field ``{evr}`` contains version information in the form of ``{epoch}:{version}-{release}``. @@ -313,9 +312,8 @@ Options Where ``{status}`` can be one of ``installed``, ``available``, ``extra``, or ``upgradable``. - The fields ``{buildtime}`` and ``{installtime}`` are in ISO 8601 format. - For example, one can parse them in Python with - ``datetime.fromisoformat()``. + The fields ``buildtime`` and ``installtime`` are in ``%Y-%m-%d + %H:%M:%S`` format in UTC. Newlines in the ``{description}`` field are replaced with pipe characters (``|``) for easier processing. @@ -353,9 +351,8 @@ Options Where ``STATUS`` can be one of ``"installed"``, ``"available"``, ``"extra"``, or ``"upgradable"``. - The fields ``buildtime`` and ``installtime`` are in ISO 8601 format. - For example, one can parse them in Python using - `datetime.fromisoformat()`. + The fields ``buildtime`` and ``installtime`` are in ``%Y-%m-%d + %H:%M:%S`` format in UTC. search ------ diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 99691a5..3ffabfa 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -149,8 +149,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_rename, mock_remove): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = b'0\0' - build_time = '2020-09-01 14:30:00+00:00' # 1598970600 - install_time = '2020-09-01 15:30:00.508230+00:00' + build_time = '2020-09-01 14:30:00' # 1598970600 + install_time = '2020-09-01 15:30:00' for key, val in [ ('name', 'test-vm'), ('epoch', '2'), @@ -280,8 +280,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_rename, mock_remove): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = b'0\0' - build_time = '2020-09-01 14:30:00+00:00' # 1598970600 - install_time = '2020-09-01 15:30:00.508230+00:00' + build_time = '2020-09-01 14:30:00' # 1598970600 + install_time = '2020-09-01 15:30:00' for key, val in [ ('name', 'test-vm'), ('epoch', '2'), diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index fcc3fdc..e3876cd 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -35,6 +35,7 @@ PACKAGE_NAME_PREFIX = 'qubes-template-' CACHE_DIR = os.path.join(xdg.BaseDirectory.xdg_cache_home, 'qvm-template') UNVERIFIED_SUFFIX = '.unverified' LOCK_FILE = '/var/tmp/qvm-template.lck' +DATE_FMT = '%Y-%m-%d %H:%M:%S' def qubes_release() -> str: """Return the Qubes release.""" @@ -287,7 +288,7 @@ def query_local(vm: qubesadmin.vm.QubesVM) -> Template: vm.features['template-release'], vm.features['template-reponame'], vm.get_disk_utilization(), - datetime.datetime.fromisoformat(vm.features['template-buildtime']), + datetime.datetime.strptime(vm.features['template-buildtime'], DATE_FMT), vm.features['template-license'], vm.features['template-url'], vm.features['template-summary'], @@ -922,11 +923,13 @@ def install( package_hdr[rpm.RPMTAG_RELEASE] tpl.features['template-reponame'] = reponame tpl.features['template-buildtime'] = \ - str(datetime.datetime.fromtimestamp( - int(package_hdr[rpm.RPMTAG_BUILDTIME]), - tz=datetime.timezone.utc)) + datetime.datetime.fromtimestamp( + int(package_hdr[rpm.RPMTAG_BUILDTIME]), + tz=datetime.timezone.utc) \ + .strftime(DATE_FMT) tpl.features['template-installtime'] = \ - str(datetime.datetime.today(tz=datetime.timezone.utc)) + datetime.datetime.today( + tz=datetime.timezone.utc).strftime(DATE_FMT) tpl.features['template-license'] = \ package_hdr[rpm.RPMTAG_LICENSE] tpl.features['template-url'] = \ @@ -1011,8 +1014,8 @@ def list_templates(args: argparse.Namespace, name, epoch, version, release, reponame, dlsize, \ buildtime, licence, url, summary, description = data dlsize = str(dlsize) - buildtime = str(buildtime) - install_time = str(install_time) if install_time else '' + buildtime = buildtime.strftime(DATE_FMT) + install_time = install_time.strftime(DATE_FMT) if install_time else '' if replace_newline: description = description.replace('\n', '|') output.append({ From 3f75e6e49ee9a23e5016fb21c1c8bc800ac0b60f Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Mon, 7 Sep 2020 00:41:03 +0800 Subject: [PATCH 069/119] qvm-template: Add tests for qrexec_payload --- qubesadmin/tests/tools/qvm_template.py | 290 +++++++++++++++++++++++++ 1 file changed, 290 insertions(+) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 3ffabfa..4109aca 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -769,3 +769,293 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock.call('/var/tmp/qvm-template.lck') ]) self.assertAllCalled() + + def test_110_qrexec_payload_refresh_success(self): + with tempfile.NamedTemporaryFile() as repo_conf1, \ + tempfile.NamedTemporaryFile() as repo_conf2: + repo_str1 = \ +'''[qubes-templates-itl] +name = Qubes Templates repository +#baseurl = https://yum.qubes-os.org/r$releasever/templates-itl +#baseurl = http://yum.qubesosfasa4zl44o4tws22di6kepyzfeqv3tg4e3ztknltfxqrymdad.onion/r$releasever/templates-itl +metalink = https://yum.qubes-os.org/r$releasever/templates-itl/repodata/repomd.xml.metalink +enabled = 1 +fastestmirror = 1 +metadata_expire = 7d +gpgcheck = 1 +gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +''' + repo_str2 = \ +'''[qubes-templates-itl-testing] +name = Qubes Templates repository +#baseurl = https://yum.qubes-os.org/r$releasever/templates-itl-testing +#baseurl = http://yum.qubesosfasa4zl44o4tws22di6kepyzfeqv3tg4e3ztknltfxqrymdad.onion/r$releasever/templates-itl-testing +metalink = https://yum.qubes-os.org/r$releasever/templates-itl-testing/repodata/repomd.xml.metalink +enabled = 0 +fastestmirror = 1 +gpgcheck = 1 +gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +''' + repo_conf1.write(repo_str1.encode()) + repo_conf1.flush() + repo_conf2.write(repo_str2.encode()) + repo_conf2.flush() + args = argparse.Namespace( + enablerepo=['repo1', 'repo2'], + disablerepo=['repo3', 'repo4', 'repo5'], + repoid=[], + releasever='4.1', + repo_files=[repo_conf1.name, repo_conf2.name] + ) + res = qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + 'qubes-template-fedora-32', True) + self.assertEqual(res, +'''--enablerepo=repo1 +--enablerepo=repo2 +--disablerepo=repo3 +--disablerepo=repo4 +--disablerepo=repo5 +--refresh +--releasever=4.1 +qubes-template-fedora-32 +--- +''' + repo_str1 + '\n' + repo_str2 + '\n') + self.assertAllCalled() + + def test_111_qrexec_payload_norefresh_success(self): + with tempfile.NamedTemporaryFile() as repo_conf1: + repo_str1 = \ +'''[qubes-templates-itl] +name = Qubes Templates repository +#baseurl = https://yum.qubes-os.org/r$releasever/templates-itl +#baseurl = http://yum.qubesosfasa4zl44o4tws22di6kepyzfeqv3tg4e3ztknltfxqrymdad.onion/r$releasever/templates-itl +metalink = https://yum.qubes-os.org/r$releasever/templates-itl/repodata/repomd.xml.metalink +enabled = 1 +fastestmirror = 1 +metadata_expire = 7d +gpgcheck = 1 +gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +''' + repo_conf1.write(repo_str1.encode()) + repo_conf1.flush() + args = argparse.Namespace( + enablerepo=[], + disablerepo=[], + repoid=['repo1', 'repo2'], + releasever='4.1', + repo_files=[repo_conf1.name] + ) + res = qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + 'qubes-template-fedora-32', False) + self.assertEqual(res, +'''--repoid=repo1 +--repoid=repo2 +--releasever=4.1 +qubes-template-fedora-32 +--- +''' + repo_str1 + '\n') + self.assertAllCalled() + + def test_112_qrexec_payload_specnewline_fail(self): + with tempfile.NamedTemporaryFile() as repo_conf1: + repo_str1 = \ +'''[qubes-templates-itl] +name = Qubes Templates repository +#baseurl = https://yum.qubes-os.org/r$releasever/templates-itl +#baseurl = http://yum.qubesosfasa4zl44o4tws22di6kepyzfeqv3tg4e3ztknltfxqrymdad.onion/r$releasever/templates-itl +metalink = https://yum.qubes-os.org/r$releasever/templates-itl/repodata/repomd.xml.metalink +enabled = 1 +fastestmirror = 1 +metadata_expire = 7d +gpgcheck = 1 +gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +''' + repo_conf1.write(repo_str1.encode()) + repo_conf1.flush() + args = argparse.Namespace( + enablerepo=[], + disablerepo=[], + repoid=['repo1', 'repo2'], + releasever='4.1', + repo_files=[repo_conf1.name] + ) + with self.assertRaises(SystemExit), \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + 'qubes-template-fedora\n-32', False) + # Check error message + self.assertTrue('Malformed template name' + in mock_err.getvalue()) + self.assertTrue("argument should not contain '\\n'" + in mock_err.getvalue()) + self.assertAllCalled() + + def test_113_qrexec_payload_enablereponewline_fail(self): + with tempfile.NamedTemporaryFile() as repo_conf1: + repo_str1 = \ +'''[qubes-templates-itl] +name = Qubes Templates repository +#baseurl = https://yum.qubes-os.org/r$releasever/templates-itl +#baseurl = http://yum.qubesosfasa4zl44o4tws22di6kepyzfeqv3tg4e3ztknltfxqrymdad.onion/r$releasever/templates-itl +metalink = https://yum.qubes-os.org/r$releasever/templates-itl/repodata/repomd.xml.metalink +enabled = 1 +fastestmirror = 1 +metadata_expire = 7d +gpgcheck = 1 +gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +''' + repo_conf1.write(repo_str1.encode()) + repo_conf1.flush() + args = argparse.Namespace( + enablerepo=['repo\n0'], + disablerepo=[], + repoid=['repo1', 'repo2'], + releasever='4.1', + repo_files=[repo_conf1.name] + ) + with self.assertRaises(SystemExit), \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + 'qubes-template-fedora-32', False) + # Check error message + self.assertTrue('Malformed --enablerepo' + in mock_err.getvalue()) + self.assertTrue("argument should not contain '\\n'" + in mock_err.getvalue()) + self.assertAllCalled() + + def test_114_qrexec_payload_disablereponewline_fail(self): + with tempfile.NamedTemporaryFile() as repo_conf1: + repo_str1 = \ +'''[qubes-templates-itl] +name = Qubes Templates repository +#baseurl = https://yum.qubes-os.org/r$releasever/templates-itl +#baseurl = http://yum.qubesosfasa4zl44o4tws22di6kepyzfeqv3tg4e3ztknltfxqrymdad.onion/r$releasever/templates-itl +metalink = https://yum.qubes-os.org/r$releasever/templates-itl/repodata/repomd.xml.metalink +enabled = 1 +fastestmirror = 1 +metadata_expire = 7d +gpgcheck = 1 +gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +''' + repo_conf1.write(repo_str1.encode()) + repo_conf1.flush() + args = argparse.Namespace( + enablerepo=[], + disablerepo=['repo\n0'], + repoid=['repo1', 'repo2'], + releasever='4.1', + repo_files=[repo_conf1.name] + ) + with self.assertRaises(SystemExit), \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + 'qubes-template-fedora-32', False) + # Check error message + self.assertTrue('Malformed --disablerepo' + in mock_err.getvalue()) + self.assertTrue("argument should not contain '\\n'" + in mock_err.getvalue()) + self.assertAllCalled() + + def test_115_qrexec_payload_repoidnewline_fail(self): + with tempfile.NamedTemporaryFile() as repo_conf1: + repo_str1 = \ +'''[qubes-templates-itl] +name = Qubes Templates repository +#baseurl = https://yum.qubes-os.org/r$releasever/templates-itl +#baseurl = http://yum.qubesosfasa4zl44o4tws22di6kepyzfeqv3tg4e3ztknltfxqrymdad.onion/r$releasever/templates-itl +metalink = https://yum.qubes-os.org/r$releasever/templates-itl/repodata/repomd.xml.metalink +enabled = 1 +fastestmirror = 1 +metadata_expire = 7d +gpgcheck = 1 +gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +''' + repo_conf1.write(repo_str1.encode()) + repo_conf1.flush() + args = argparse.Namespace( + enablerepo=[], + disablerepo=[], + repoid=['repo\n1', 'repo2'], + releasever='4.1', + repo_files=[repo_conf1.name] + ) + with self.assertRaises(SystemExit), \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + 'qubes-template-fedora-32', False) + # Check error message + self.assertTrue('Malformed --repoid' + in mock_err.getvalue()) + self.assertTrue("argument should not contain '\\n'" + in mock_err.getvalue()) + self.assertAllCalled() + + def test_116_qrexec_payload_releasevernewline_fail(self): + with tempfile.NamedTemporaryFile() as repo_conf1: + repo_str1 = \ +'''[qubes-templates-itl] +name = Qubes Templates repository +#baseurl = https://yum.qubes-os.org/r$releasever/templates-itl +#baseurl = http://yum.qubesosfasa4zl44o4tws22di6kepyzfeqv3tg4e3ztknltfxqrymdad.onion/r$releasever/templates-itl +metalink = https://yum.qubes-os.org/r$releasever/templates-itl/repodata/repomd.xml.metalink +enabled = 1 +fastestmirror = 1 +metadata_expire = 7d +gpgcheck = 1 +gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +''' + repo_conf1.write(repo_str1.encode()) + repo_conf1.flush() + args = argparse.Namespace( + enablerepo=[], + disablerepo=[], + repoid=['repo1', 'repo2'], + releasever='4\n.1', + repo_files=[repo_conf1.name] + ) + with self.assertRaises(SystemExit), \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + 'qubes-template-fedora-32', False) + # Check error message + self.assertTrue('Malformed --releasever' + in mock_err.getvalue()) + self.assertTrue("argument should not contain '\\n'" + in mock_err.getvalue()) + self.assertAllCalled() + + def test_117_qrexec_payload_specdash_fail(self): + with tempfile.NamedTemporaryFile() as repo_conf1: + repo_str1 = \ +'''[qubes-templates-itl] +name = Qubes Templates repository +#baseurl = https://yum.qubes-os.org/r$releasever/templates-itl +#baseurl = http://yum.qubesosfasa4zl44o4tws22di6kepyzfeqv3tg4e3ztknltfxqrymdad.onion/r$releasever/templates-itl +metalink = https://yum.qubes-os.org/r$releasever/templates-itl/repodata/repomd.xml.metalink +enabled = 1 +fastestmirror = 1 +metadata_expire = 7d +gpgcheck = 1 +gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +''' + repo_conf1.write(repo_str1.encode()) + repo_conf1.flush() + args = argparse.Namespace( + enablerepo=[], + disablerepo=[], + repoid=['repo1', 'repo2'], + releasever='4.1', + repo_files=[repo_conf1.name] + ) + with self.assertRaises(SystemExit), \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + '---', False) + # Check error message + self.assertTrue('Malformed template name' + in mock_err.getvalue()) + self.assertTrue("argument should not be '---'" + in mock_err.getvalue()) + self.assertAllCalled() From 5e1e0daa5c59525cab18cff9605422f46cd14d37 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Mon, 7 Sep 2020 01:18:59 +0800 Subject: [PATCH 070/119] Make TestProcess.communicate return str instead of IO object --- qubesadmin/tests/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/qubesadmin/tests/__init__.py b/qubesadmin/tests/__init__.py index 787c3d9..a0fa1e6 100644 --- a/qubesadmin/tests/__init__.py +++ b/qubesadmin/tests/__init__.py @@ -60,11 +60,13 @@ class TestProcess(object): self.stdin_close = self.stdin.close self.stdin.close = self.store_input self.stdin.flush = self.store_input - if stdout == subprocess.PIPE: + if stdout == subprocess.PIPE or stdout == subprocess.DEVNULL \ + or stdout is None: self.stdout = io.BytesIO() else: self.stdout = stdout - if stderr == subprocess.PIPE: + if stderr == subprocess.PIPE or stderr == subprocess.DEVNULL \ + or stderr is None: self.stderr = io.BytesIO() else: self.stderr = stderr @@ -82,7 +84,7 @@ class TestProcess(object): self.stdin.write(input) self.stdin.close() self.stdin_close() - return self.stdout, self.stderr + return self.stdout.read(), self.stderr.read() def wait(self): self.stdin_close() From 3fac2097ebc8d93c60d5da9513d150a089274864 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Mon, 7 Sep 2020 01:52:56 +0800 Subject: [PATCH 071/119] qvm-template: Add partial tests for qrexec_repoquery --- qubesadmin/tests/tools/qvm_template.py | 202 +++++++++++++++++++++++++ qubesadmin/tools/qvm_template.py | 3 +- 2 files changed, 204 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 4109aca..e8a55d8 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -1059,3 +1059,205 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev self.assertTrue("argument should not be '---'" in mock_err.getvalue()) self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_120_qrexec_repoquery_success(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateSearch')] = \ +b'''qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template\n for fedora-32\n| +qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| +''' + res = qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32') + self.assertEqual(res, [ + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '1', + '4.2', + '20200201', + 'qubes-templates-itl-testing', + 2048576, + datetime.datetime(2020, 2, 23, 4, 56), + 'GPLv2', + 'https://qubes-os.org/?', + 'Qubes template for fedora-32 v2', + 'Qubes template\n for fedora-32 v2\n' + ) + ]) + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', False) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_121_qrexec_repoquery_refresh_success(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateSearch')] = \ +b'''qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template\n for fedora-32\n| +qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| +''' + res = qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32', True) + self.assertEqual(res, [ + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '1', + '4.2', + '20200201', + 'qubes-templates-itl-testing', + 2048576, + datetime.datetime(2020, 2, 23, 4, 56), + 'GPLv2', + 'https://qubes-os.org/?', + 'Qubes template for fedora-32 v2', + 'Qubes template\n for fedora-32 v2\n' + ) + ]) + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', True) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_122_qrexec_repoquery_ignorenonspec_success(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateSearch')] = \ +b'''qubes-template-debian-10|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for debian-10|Qubes template for debian-10\n| +qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template for fedora-32\n| +''' + res = qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32') + self.assertEqual(res, [ + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template for fedora-32\n' + ) + ]) + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', False) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_123_qrexec_repoquery_ignorebadname_success(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateSearch')] = \ +b'''template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| +qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template for fedora-32\n| +''' + res = qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32') + self.assertEqual(res, [ + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template for fedora-32\n' + ) + ]) + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', False) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_124_qrexec_repoquery_searchfail_fail(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + with mock.patch('qubesadmin.tests.TestProcess.wait') \ + as mock_wait: + mock_wait.return_value = 1 + with self.assertRaises(ConnectionError): + qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32') + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', False) + ]) + self.assertAllCalled() + + # TODO: Also test feeding broken data to qrexec_repoquery diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index e3876cd..06c7f3d 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -483,7 +483,8 @@ def qrexec_repoquery( if not re.fullmatch(licence_re, licence): raise ValueError # Check name actually matches spec - if not is_match_spec(name, epoch, version, release, spec): + if not is_match_spec(PACKAGE_NAME_PREFIX + name, + epoch, version, release, spec)[0]: continue result.append(Template(name, epoch, version, release, reponame, From 554459ef42c7b9c23ef0e8c5eb2921185eb308de Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 5 Sep 2020 02:06:53 +0800 Subject: [PATCH 072/119] qvm-template: Fix compatibility with Python 3.6 --- qubesadmin/tools/qvm_template.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 06c7f3d..dc4f8ab 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -59,7 +59,7 @@ def parser_gen() -> argparse.ArgumentParser: formatter = argparse.ArgumentDefaultsHelpFormatter parser_main = argparse.ArgumentParser(description='Qubes Template Manager', formatter_class=formatter) - subparsers = parser_main.add_subparsers(dest='operation', required=True, + subparsers = parser_main.add_subparsers(dest='operation', description='Command to run.') def parser_add_command(cmd, help_str): @@ -1367,6 +1367,9 @@ def main(args: typing.Optional[typing.Sequence[str]] = None, """ p_args = parser.parse_args(args) + if not p_args.operation: + parser.error('An operation needs to be specified.') + # If the user specified other repo files... if len(p_args.repo_files) > 1: # ...remove the default entry From 161ff01d7d548ccbc0e6c236352df2333e8b1761 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Mon, 7 Sep 2020 02:12:03 +0800 Subject: [PATCH 073/119] qvm-template: Fix compatibility with Python 3.6 in tests --- qubesadmin/tests/tools/qvm_template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index e8a55d8..6659a7c 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -183,7 +183,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_call.side_effect = self.add_new_vm_side_effect mock_time = mock.Mock(wraps=datetime.datetime) mock_time.today.return_value = \ - datetime.datetime.fromisoformat(install_time) + datetime.datetime(2020, 9, 1, 15, 30, tzinfo=datetime.timezone.utc) with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ mock.patch('datetime.datetime', new=mock_time), \ mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ @@ -314,7 +314,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_call.side_effect = self.add_new_vm_side_effect mock_time = mock.Mock(wraps=datetime.datetime) mock_time.today.return_value = \ - datetime.datetime.fromisoformat(install_time) + datetime.datetime(2020, 9, 1, 15, 30, tzinfo=datetime.timezone.utc) with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ mock.patch('datetime.datetime', new=mock_time), \ mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ From b9f1d4c6332aa1ddf9f5b9c1af9302c19d7033df Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Mon, 7 Sep 2020 02:12:22 +0800 Subject: [PATCH 074/119] qvm-template: Make pylint happy --- qubesadmin/tools/qvm_template.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index dc4f8ab..a74c29b 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -1016,7 +1016,8 @@ def list_templates(args: argparse.Namespace, buildtime, licence, url, summary, description = data dlsize = str(dlsize) buildtime = buildtime.strftime(DATE_FMT) - install_time = install_time.strftime(DATE_FMT) if install_time else '' + install_time = install_time.strftime(DATE_FMT) \ + if install_time else '' if replace_newline: description = description.replace('\n', '|') output.append({ From dc26ba0ebf1bc61f7d2707a6916676494095f65d Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Wed, 9 Sep 2020 03:04:37 +0800 Subject: [PATCH 075/119] qvm-template: Add tests for qrexec_repoquery and get_dl_list --- qubesadmin/tests/tools/qvm_template.py | 932 ++++++++++++++++++++++++- 1 file changed, 898 insertions(+), 34 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 6659a7c..8204059 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -680,10 +680,10 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): '/var/tmp/qvm-template-tmpdir' pathlib.Path('/var/tmp/qvm-template.lck').touch() try: - with self.assertRaises(SystemExit), \ - mock.patch('os.remove') as mock_remove: - qubesadmin.tools.qvm_template.install(args, self.app) - self.assertEqual(mock_remove.mock_calls, []) + with mock.patch('os.remove') as mock_remove: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.install(args, self.app) + self.assertEqual(mock_remove.mock_calls, []) finally: # Lock file not removed self.assertTrue(os.path.exists('/var/tmp/qvm-template.lck')) @@ -879,10 +879,10 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev releasever='4.1', repo_files=[repo_conf1.name] ) - with self.assertRaises(SystemExit), \ - mock.patch('sys.stderr', new=io.StringIO()) as mock_err: - qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, - 'qubes-template-fedora\n-32', False) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + 'qubes-template-fedora\n-32', False) # Check error message self.assertTrue('Malformed template name' in mock_err.getvalue()) @@ -913,10 +913,10 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev releasever='4.1', repo_files=[repo_conf1.name] ) - with self.assertRaises(SystemExit), \ - mock.patch('sys.stderr', new=io.StringIO()) as mock_err: - qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, - 'qubes-template-fedora-32', False) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + 'qubes-template-fedora-32', False) # Check error message self.assertTrue('Malformed --enablerepo' in mock_err.getvalue()) @@ -947,15 +947,15 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev releasever='4.1', repo_files=[repo_conf1.name] ) - with self.assertRaises(SystemExit), \ - mock.patch('sys.stderr', new=io.StringIO()) as mock_err: - qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, - 'qubes-template-fedora-32', False) - # Check error message - self.assertTrue('Malformed --disablerepo' - in mock_err.getvalue()) - self.assertTrue("argument should not contain '\\n'" - in mock_err.getvalue()) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + 'qubes-template-fedora-32', False) + # Check error message + self.assertTrue('Malformed --disablerepo' + in mock_err.getvalue()) + self.assertTrue("argument should not contain '\\n'" + in mock_err.getvalue()) self.assertAllCalled() def test_115_qrexec_payload_repoidnewline_fail(self): @@ -981,10 +981,10 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev releasever='4.1', repo_files=[repo_conf1.name] ) - with self.assertRaises(SystemExit), \ - mock.patch('sys.stderr', new=io.StringIO()) as mock_err: - qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, - 'qubes-template-fedora-32', False) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + 'qubes-template-fedora-32', False) # Check error message self.assertTrue('Malformed --repoid' in mock_err.getvalue()) @@ -1015,10 +1015,10 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev releasever='4\n.1', repo_files=[repo_conf1.name] ) - with self.assertRaises(SystemExit), \ - mock.patch('sys.stderr', new=io.StringIO()) as mock_err: - qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, - 'qubes-template-fedora-32', False) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + 'qubes-template-fedora-32', False) # Check error message self.assertTrue('Malformed --releasever' in mock_err.getvalue()) @@ -1049,10 +1049,10 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev releasever='4.1', repo_files=[repo_conf1.name] ) - with self.assertRaises(SystemExit), \ - mock.patch('sys.stderr', new=io.StringIO()) as mock_err: - qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, - '---', False) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, + '---', False) # Check error message self.assertTrue('Malformed template name' in mock_err.getvalue()) @@ -1260,4 +1260,868 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 ]) self.assertAllCalled() - # TODO: Also test feeding broken data to qrexec_repoquery + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_125_qrexec_repoquery_extrafield_fail(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateSearch')] = \ +b'''qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Extra field|Qubes template\n for fedora-32 v2\n| +qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template for fedora-32\n| +''' + with self.assertRaisesRegex(ConnectionError, + "unexpected data format"): + qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32') + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', False) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_125_qrexec_repoquery_missingfield_fail(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateSearch')] = \ +b'''qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| +qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template for fedora-32\n| +''' + with self.assertRaisesRegex(ConnectionError, + "unexpected data format"): + qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32') + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', False) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_126_qrexec_repoquery_badfieldname_fail(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateSearch')] = \ +b'''qubes-template-fedora-(32)|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| +qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template for fedora-32\n| +''' + with self.assertRaisesRegex(ConnectionError, + "unexpected data format"): + qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32') + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', False) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_126_qrexec_repoquery_badfieldepoch_fail(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateSearch')] = \ +b'''qubes-template-fedora-32|!1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| +qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template for fedora-32\n| +''' + with self.assertRaisesRegex(ConnectionError, + "unexpected data format"): + qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32') + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', False) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_126_qrexec_repoquery_badfieldreponame_fail(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateSearch')] = \ +b'''qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| +qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template for fedora-32\n| +''' + with self.assertRaisesRegex(ConnectionError, + "unexpected data format"): + qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32') + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', False) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_126_qrexec_repoquery_badfielddlsize_fail(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateSearch')] = \ +b'''qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048a576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| +qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template for fedora-32\n| +''' + with self.assertRaisesRegex(ConnectionError, + "unexpected data format"): + qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32') + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', False) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_126_qrexec_repoquery_badfielddate_fail(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateSearch')] = \ +b'''qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| +qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template for fedora-32\n| +''' + with self.assertRaisesRegex(ConnectionError, + "unexpected data format"): + qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32') + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', False) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_payload') + def test_126_qrexec_repoquery_license_fail(self, mock_payload): + args = argparse.Namespace(updatevm='test-vm') + mock_payload.return_value = 'str1\nstr2' + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateSearch')] = \ +b'''qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2:)|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| +qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template for fedora-32\n| +''' + with self.assertRaisesRegex(ConnectionError, + "unexpected data format"): + qubesadmin.tools.qvm_template.qrexec_repoquery(args, self.app, + 'qubes-template-fedora-32') + self.assertEqual(self.app.service_calls, [ + ('test-vm', 'qubes.TemplateSearch', + {'filter_esc': True, 'stdout': subprocess.PIPE}), + ('test-vm', 'qubes.TemplateSearch', b'str1\nstr2') + ]) + self.assertEqual(mock_payload.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32', False) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_130_get_dl_list_latest_success(self, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.2', + '20200201', + 'qubes-templates-itl-testing', + 2048576, + datetime.datetime(2020, 2, 23, 4, 56), + 'GPLv2', + 'https://qubes-os.org/?', + 'Qubes template for fedora-32 v2', + 'Qubes template\n for fedora-32 v2\n' + ) + ] + args = argparse.Namespace( + templates=['some.local.file.rpm', 'fedora-32'] + ) + ret = qubesadmin.tools.qvm_template.get_dl_list(args, self.app) + self.assertEqual(ret, { + 'fedora-32': qubesadmin.tools.qvm_template.DlEntry( + ('1', '4.1', '20200101'), 'qubes-templates-itl', 1048576) + }) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_131_get_dl_list_latest_notfound_fail(self, mock_query): + mock_query.return_value = [] + args = argparse.Namespace( + templates=['some.local.file.rpm', 'fedora-31'] + ) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.get_dl_list(args, self.app) + self.assertTrue('not found' in mock_err.getvalue()) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-31') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_132_get_dl_list_multimerge0_success(self, mock_query): + counter = 0 + def f(*args): + nonlocal counter + counter += 1 + if counter == 1: + return [ + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.2', + '20200201', + 'qubes-templates-itl-testing', + 2048576, + datetime.datetime(2020, 2, 23, 4, 56), + 'GPLv2', + 'https://qubes-os.org/?', + 'Qubes template for fedora-32 v2', + 'Qubes template\n for fedora-32 v2\n' + ) + ] + return [ + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ) + ] + mock_query.side_effect = f + args = argparse.Namespace( + templates=['some.local.file.rpm', 'fedora-32:0', 'fedora-32:1'] + ) + ret = qubesadmin.tools.qvm_template.get_dl_list(args, self.app) + self.assertEqual(ret, { + 'fedora-32': qubesadmin.tools.qvm_template.DlEntry( + ('1', '4.1', '20200101'), 'qubes-templates-itl', 1048576) + }) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32:0'), + mock.call(args, self.app, 'qubes-template-fedora-32:1') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_132_get_dl_list_multimerge1_success(self, mock_query): + counter = 0 + def f(*args): + nonlocal counter + counter += 1 + if counter == 1: + return [ + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '2', + '4.2', + '20200201', + 'qubes-templates-itl-testing', + 2048576, + datetime.datetime(2020, 2, 23, 4, 56), + 'GPLv2', + 'https://qubes-os.org/?', + 'Qubes template for fedora-32 v2', + 'Qubes template\n for fedora-32 v2\n' + ) + ] + return [ + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ) + ] + mock_query.side_effect = f + args = argparse.Namespace( + templates=['some.local.file.rpm', 'fedora-32:2', 'fedora-32:1'] + ) + ret = qubesadmin.tools.qvm_template.get_dl_list(args, self.app) + self.assertEqual(ret, { + 'fedora-32': qubesadmin.tools.qvm_template.DlEntry( + ('2', '4.2', '20200201'), 'qubes-templates-itl-testing', 2048576) + }) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32:2'), + mock.call(args, self.app, 'qubes-template-fedora-32:1') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_133_get_dl_list_reinstall_success(self, mock_query): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-name', + None)] = b'0\0test-vm' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-epoch', + None)] = b'0\x000' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-version', + None)] = b'0\x004.2' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-release', + None)] = b'0\x0020200201' + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for test-vm\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '0', + '4.2', + '20200201', + 'qubes-templates-itl-testing', + 2048576, + datetime.datetime(2020, 2, 23, 4, 56), + 'GPLv2', + 'https://qubes-os.org/?', + 'Qubes template for test-vm v2', + 'Qubes template\n for test-vm v2\n' + ) + ] + args = argparse.Namespace( + templates=['some.local.file.rpm', 'test-vm'] + ) + ret = qubesadmin.tools.qvm_template.get_dl_list(args, self.app, + qubesadmin.tools.qvm_template.VersionSelector.REINSTALL) + self.assertEqual(ret, { + 'test-vm': qubesadmin.tools.qvm_template.DlEntry( + ('0', '4.2', '20200201'), + 'qubes-templates-itl-testing', + 2048576 + ) + }) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-test-vm') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_134_get_dl_list_reinstall_nolocal_fail(self, mock_query): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00' + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for test-vm\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '0', + '4.2', + '20200201', + 'qubes-templates-itl-testing', + 2048576, + datetime.datetime(2020, 2, 23, 4, 56), + 'GPLv2', + 'https://qubes-os.org/?', + 'Qubes template for test-vm v2', + 'Qubes template\n for test-vm v2\n' + ) + ] + args = argparse.Namespace(templates=['test-vm']) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.get_dl_list(args, self.app, + qubesadmin.tools.qvm_template.VersionSelector.REINSTALL) + self.assertTrue('not already installed' in mock_err.getvalue()) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-test-vm') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_135_get_dl_list_reinstall_nonmanaged_fail(self, mock_query): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for test-vm\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '0', + '4.2', + '20200201', + 'qubes-templates-itl-testing', + 2048576, + datetime.datetime(2020, 2, 23, 4, 56), + 'GPLv2', + 'https://qubes-os.org/?', + 'Qubes template for test-vm v2', + 'Qubes template\n for test-vm v2\n' + ) + ] + args = argparse.Namespace(templates=['test-vm']) + def qubesd_call(dest, method, + arg=None, payload=None, payload_stream=None, + orig_func=self.app.qubesd_call): + if method == 'admin.vm.feature.Get': + raise KeyError + return orig_func(dest, method, arg, payload, payload_stream) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ + mock.patch.object(self.app, 'qubesd_call') as mock_call: + mock_call.side_effect = qubesd_call + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.get_dl_list(args, self.app, + qubesadmin.tools.qvm_template.VersionSelector.REINSTALL) + self.assertTrue('not managed' in mock_err.getvalue()) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-test-vm') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_135_get_dl_list_reinstall_nonmanagednoname_fail(self, mock_query): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-name', + None)] = b'0\0test-vm-2' + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for test-vm\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '0', + '4.2', + '20200201', + 'qubes-templates-itl-testing', + 2048576, + datetime.datetime(2020, 2, 23, 4, 56), + 'GPLv2', + 'https://qubes-os.org/?', + 'Qubes template for test-vm v2', + 'Qubes template\n for test-vm v2\n' + ) + ] + args = argparse.Namespace(templates=['test-vm']) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.get_dl_list(args, self.app, + qubesadmin.tools.qvm_template.VersionSelector.REINSTALL) + self.assertTrue('not managed' in mock_err.getvalue()) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-test-vm') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_136_get_dl_list_downgrade_success(self, mock_query): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-name', + None)] = b'0\0test-vm' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-epoch', + None)] = b'0\x000' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-version', + None)] = b'0\x004.3' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-release', + None)] = b'0\x0020200201' + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '0', + '4.2', + '20200201', + 'qubes-templates-itl-testing', + 2048576, + datetime.datetime(2020, 2, 23, 4, 56), + 'GPLv2', + 'https://qubes-os.org/?', + 'Qubes template for test-vm v2', + 'Qubes template\n for test-vm v2\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for test-vm\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for test-vm\n' + ) + ] + args = argparse.Namespace(templates=['test-vm']) + ret = qubesadmin.tools.qvm_template.get_dl_list(args, self.app, + qubesadmin.tools.qvm_template.VersionSelector.LATEST_LOWER) + self.assertEqual(ret, { + 'test-vm': qubesadmin.tools.qvm_template.DlEntry( + ('0', '4.2', '20200201'), + 'qubes-templates-itl-testing', + 2048576 + ) + }) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-test-vm') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_137_get_dl_list_downgrade_nonmanaged_fail(self, mock_query): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-name', + None)] = b'0\0test-vm-2' + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '0', + '4.2', + '20200201', + 'qubes-templates-itl-testing', + 2048576, + datetime.datetime(2020, 2, 23, 4, 56), + 'GPLv2', + 'https://qubes-os.org/?', + 'Qubes template for test-vm v2', + 'Qubes template\n for test-vm v2\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for test-vm\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for test-vm\n' + ) + ] + args = argparse.Namespace(templates=['test-vm']) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.get_dl_list(args, self.app, + qubesadmin.tools.qvm_template.VersionSelector.REINSTALL) + self.assertTrue('not managed' in mock_err.getvalue()) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-test-vm') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_138_get_dl_list_downgrade_notfound_skip(self, mock_query): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-name', + None)] = b'0\0test-vm' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-epoch', + None)] = b'0\x000' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-version', + None)] = b'0\x004.3' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-release', + None)] = b'0\x0020200201' + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for test-vm\n' + ) + ] + args = argparse.Namespace(templates=['test-vm']) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + ret = qubesadmin.tools.qvm_template.get_dl_list(args, self.app, + qubesadmin.tools.qvm_template.VersionSelector.LATEST_LOWER) + self.assertTrue('lowest version' in mock_err.getvalue()) + self.assertEqual(ret, {}) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-test-vm') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_139_get_dl_list_upgrade_success(self, mock_query): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-name', + None)] = b'0\0test-vm' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-epoch', + None)] = b'0\x000' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-version', + None)] = b'0\x004.3' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-release', + None)] = b'0\x0020200201' + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for test-vm\n' + ) + ] + args = argparse.Namespace(templates=['test-vm']) + ret = qubesadmin.tools.qvm_template.get_dl_list(args, self.app, + qubesadmin.tools.qvm_template.VersionSelector.LATEST_HIGHER) + self.assertEqual(ret, { + 'test-vm': qubesadmin.tools.qvm_template.DlEntry( + ('1', '4.1', '20200101'), 'qubes-templates-itl', 1048576 + ) + }) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-test-vm') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_140_get_dl_list_downgrade_notfound_skip(self, mock_query): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=AppVM state=Halted\n' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-name', + None)] = b'0\0test-vm' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-epoch', + None)] = b'0\x000' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-version', + None)] = b'0\x004.3' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-release', + None)] = b'0\x0020200201' + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for test-vm\n' + ) + ] + args = argparse.Namespace(templates=['test-vm']) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + ret = qubesadmin.tools.qvm_template.get_dl_list(args, self.app, + qubesadmin.tools.qvm_template.VersionSelector.LATEST_HIGHER) + self.assertTrue('highest version' in mock_err.getvalue()) + self.assertEqual(ret, {}) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-test-vm') + ]) + self.assertAllCalled() From 7facc7d35ff261aff671721fb5036da0efc7f46e Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 15 Sep 2020 00:22:05 +0800 Subject: [PATCH 076/119] qvm-template: Fix minor bugs * Incomprehensive spec filtering in `list_templates` * Type error of `install_time` in `list_templates` * Incorrect version comparision in `search` --- qubesadmin/tools/qvm_template.py | 37 +++++++++++++++++--------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index a74c29b..99d381b 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -448,6 +448,7 @@ def qrexec_repoquery( date_re = re.compile(r'^\d+-\d+-\d+ \d+:\d+$') licence_re = re.compile(r'^[A-Za-z0-9._+\-()]*$') result = [] + # FIXME: This breaks when \n is the first character of the description for line in stdout.split('|\n'): # Note that there's an empty entry at the end as .strip() is not used. # This is because if .strip() is used, the .split() will not work. @@ -575,8 +576,8 @@ def verify_rpm( except rpm.error as e: if str(e) == 'public key not trusted' \ or str(e) == 'public key not available': - # TODO: This does not work - # Should just tell TransactionSet not to verify sigs + # FIXME: This does not work + # Should just tell TransactionSet not to verify sigs return hdr if nogpgcheck else None return None return hdr @@ -896,7 +897,7 @@ def install( for rpmfile, reponame, name, package_hdr in verified_rpm_list: with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: print('Installing template \'%s\'...' % name, file=sys.stderr) - # TODO: Handle return value + # FIXME: Handle return value extract_rpm(name, rpmfile, target) cmdline = [ 'qvm-template-postprocess', @@ -1016,8 +1017,7 @@ def list_templates(args: argparse.Namespace, buildtime, licence, url, summary, description = data dlsize = str(dlsize) buildtime = buildtime.strftime(DATE_FMT) - install_time = install_time.strftime(DATE_FMT) \ - if install_time else '' + install_time = install_time if install_time else '' if replace_newline: description = description.replace('\n', '|') output.append({ @@ -1046,6 +1046,11 @@ def list_templates(args: argparse.Namespace, def append_vm(vm, status): append(query_local(vm), status, vm.features['template-installtime']) + def check_append(name, evr): + return not args.templates or \ + any(is_match_spec(name, *evr, spec)[0] + for spec in args.templates) + if not (args.installed or args.available or args.extras or args.upgrades): args.all = True @@ -1060,16 +1065,12 @@ def list_templates(args: argparse.Namespace, if args.installed or args.all: for vm in app.domains: - if is_managed_template(vm): - if not args.templates or \ - any(is_match_spec( - vm.name, - *query_local_evr(vm), - spec)[0] - for spec in args.templates): + if is_managed_template(vm) and \ + check_append(vm.name, query_local_evr(vm)): append_vm(vm, TemplateState.INSTALLED) if args.available or args.all: + # Spec should already be checked by repoquery for data in query_res: append(data, TemplateState.AVAILABLE) @@ -1078,7 +1079,8 @@ def list_templates(args: argparse.Namespace, for data in query_res: remote.add(data.name) for vm in app.domains: - if is_managed_template(vm) and vm.name not in remote: + if is_managed_template(vm) and vm.name not in remote and \ + check_append(vm.name, query_local_evr(vm)): append_vm(vm, TemplateState.EXTRA) if args.upgrades: @@ -1086,10 +1088,11 @@ def list_templates(args: argparse.Namespace, for vm in app.domains: if is_managed_template(vm): local[vm.name] = query_local_evr(vm) + # Spec should already be checked by repoquery for entry in query_res: + evr = (entry.epoch, entry.version, entry.release) if entry.name in local: - if rpm.labelCompare(local[entry.name], - (entry.epoch, entry.version, entry.release)) < 0: + if rpm.labelCompare(local[entry.name], evr) < 0: append(entry, TemplateState.UPGRADABLE) if len(tpl_list) == 0: @@ -1134,7 +1137,7 @@ def search(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: query_res_tmp = [] for _, grp in itertools.groupby(sorted(query_res), lambda x: x[0]): def compare(lhs, rhs): - return lhs if rpm.labelCompare(lhs[1:4], rhs[1:4]) < 0 else rhs + return lhs if rpm.labelCompare(lhs[1:4], rhs[1:4]) > 0 else rhs query_res_tmp.append(functools.reduce(compare, grp)) query_res = query_res_tmp @@ -1292,7 +1295,7 @@ def clean(args: argparse.Namespace, app: qubesadmin.app.QubesBase) -> None: :param args: Arguments received by the application. :param app: Qubes application object """ - # TODO: More fine-grained options + # TODO: More fine-grained options? _ = app # unused shutil.rmtree(args.cachedir) From 20443d5c6f0d9c549d233530bc863bb1d8736135 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 15 Sep 2020 00:25:49 +0800 Subject: [PATCH 077/119] qvm-template: Add tests for functions list_templates and search --- qubesadmin/tests/tools/qvm_template.py | 1201 +++++++++++++++++++++++- 1 file changed, 1167 insertions(+), 34 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 8204059..f2ea27b 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -15,6 +15,7 @@ import qubesadmin.tools.qvm_template class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): def setUp(self): + self.maxDiff = 1e9 super().setUp() def tearDown(self): @@ -881,8 +882,8 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev ) with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: with self.assertRaises(SystemExit): - qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, - 'qubes-template-fedora\n-32', False) + qubesadmin.tools.qvm_template.qrexec_payload(args, + self.app, 'qubes-template-fedora\n-32', False) # Check error message self.assertTrue('Malformed template name' in mock_err.getvalue()) @@ -915,8 +916,8 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev ) with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: with self.assertRaises(SystemExit): - qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, - 'qubes-template-fedora-32', False) + qubesadmin.tools.qvm_template.qrexec_payload(args, + self.app, 'qubes-template-fedora-32', False) # Check error message self.assertTrue('Malformed --enablerepo' in mock_err.getvalue()) @@ -949,8 +950,8 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev ) with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: with self.assertRaises(SystemExit): - qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, - 'qubes-template-fedora-32', False) + qubesadmin.tools.qvm_template.qrexec_payload(args, + self.app, 'qubes-template-fedora-32', False) # Check error message self.assertTrue('Malformed --disablerepo' in mock_err.getvalue()) @@ -983,8 +984,8 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev ) with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: with self.assertRaises(SystemExit): - qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, - 'qubes-template-fedora-32', False) + qubesadmin.tools.qvm_template.qrexec_payload(args, + self.app, 'qubes-template-fedora-32', False) # Check error message self.assertTrue('Malformed --repoid' in mock_err.getvalue()) @@ -1017,8 +1018,8 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev ) with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: with self.assertRaises(SystemExit): - qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, - 'qubes-template-fedora-32', False) + qubesadmin.tools.qvm_template.qrexec_payload(args, + self.app, 'qubes-template-fedora-32', False) # Check error message self.assertTrue('Malformed --releasever' in mock_err.getvalue()) @@ -1051,8 +1052,8 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev ) with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: with self.assertRaises(SystemExit): - qubesadmin.tools.qvm_template.qrexec_payload(args, self.app, - '---', False) + qubesadmin.tools.qvm_template.qrexec_payload(args, + self.app, '---', False) # Check error message self.assertTrue('Malformed template name' in mock_err.getvalue()) @@ -1065,7 +1066,7 @@ gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasev args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_service_calls[ ('test-vm', 'qubes.TemplateSearch')] = \ b'''qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template\n for fedora-32\n| @@ -1116,7 +1117,7 @@ qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020 args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_service_calls[ ('test-vm', 'qubes.TemplateSearch')] = \ b'''qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 04:56|GPL|https://qubes-os.org|Qubes template for fedora-32|Qubes template\n for fedora-32\n| @@ -1167,7 +1168,7 @@ qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020 args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_service_calls[ ('test-vm', 'qubes.TemplateSearch')] = \ b'''qubes-template-debian-10|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for debian-10|Qubes template for debian-10\n| @@ -1205,7 +1206,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_service_calls[ ('test-vm', 'qubes.TemplateSearch')] = \ b'''template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| @@ -1243,7 +1244,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' with mock.patch('qubesadmin.tests.TestProcess.wait') \ as mock_wait: mock_wait.return_value = 1 @@ -1265,7 +1266,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_service_calls[ ('test-vm', 'qubes.TemplateSearch')] = \ b'''qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Extra field|Qubes template\n for fedora-32 v2\n| @@ -1290,7 +1291,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_service_calls[ ('test-vm', 'qubes.TemplateSearch')] = \ b'''qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| @@ -1315,7 +1316,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_service_calls[ ('test-vm', 'qubes.TemplateSearch')] = \ b'''qubes-template-fedora-(32)|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| @@ -1340,7 +1341,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_service_calls[ ('test-vm', 'qubes.TemplateSearch')] = \ b'''qubes-template-fedora-32|!1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| @@ -1365,7 +1366,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_service_calls[ ('test-vm', 'qubes.TemplateSearch')] = \ b'''qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-|2048576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| @@ -1390,7 +1391,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_service_calls[ ('test-vm', 'qubes.TemplateSearch')] = \ b'''qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048a576|2020-02-23 04:56|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| @@ -1415,7 +1416,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_service_calls[ ('test-vm', 'qubes.TemplateSearch')] = \ b'''qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23|GPLv2|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| @@ -1440,7 +1441,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 args = argparse.Namespace(updatevm='test-vm') mock_payload.return_value = 'str1\nstr2' self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_service_calls[ ('test-vm', 'qubes.TemplateSearch')] = \ b'''qubes-template-fedora-32|1|4.2|20200201|qubes-templates-itl-testing|2048576|2020-02-23 04:56|GPLv2:)|https://qubes-os.org/?|Qubes template for fedora-32 v2|Qubes template\n for fedora-32 v2\n| @@ -1614,7 +1615,9 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 ret = qubesadmin.tools.qvm_template.get_dl_list(args, self.app) self.assertEqual(ret, { 'fedora-32': qubesadmin.tools.qvm_template.DlEntry( - ('2', '4.2', '20200201'), 'qubes-templates-itl-testing', 2048576) + ('2', '4.2', '20200201'), + 'qubes-templates-itl-testing', + 2048576) }) self.assertEqual(mock_query.mock_calls, [ mock.call(args, self.app, 'qubes-template-fedora-32:2'), @@ -1625,7 +1628,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') def test_133_get_dl_list_reinstall_success(self, mock_query): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_calls[( 'test-vm', 'admin.vm.feature.Get', @@ -1737,7 +1740,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') def test_135_get_dl_list_reinstall_nonmanaged_fail(self, mock_query): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' mock_query.return_value = [ qubesadmin.tools.qvm_template.Template( 'test-vm', @@ -1788,7 +1791,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') def test_135_get_dl_list_reinstall_nonmanagednoname_fail(self, mock_query): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_calls[( 'test-vm', 'admin.vm.feature.Get', @@ -1836,7 +1839,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') def test_136_get_dl_list_downgrade_success(self, mock_query): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_calls[( 'test-vm', 'admin.vm.feature.Get', @@ -1916,7 +1919,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') def test_137_get_dl_list_downgrade_nonmanaged_fail(self, mock_query): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_calls[( 'test-vm', 'admin.vm.feature.Get', @@ -1977,7 +1980,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') def test_138_get_dl_list_downgrade_notfound_skip(self, mock_query): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_calls[( 'test-vm', 'admin.vm.feature.Get', @@ -2027,7 +2030,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') def test_139_get_dl_list_upgrade_success(self, mock_query): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_calls[( 'test-vm', 'admin.vm.feature.Get', @@ -2079,7 +2082,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') def test_140_get_dl_list_downgrade_notfound_skip(self, mock_query): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ - b'0\x00test-vm class=AppVM state=Halted\n' + b'0\x00test-vm class=TemplateVM state=Halted\n' self.app.expected_calls[( 'test-vm', 'admin.vm.feature.Get', @@ -2125,3 +2128,1133 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 mock.call(args, self.app, 'qubes-template-test-vm') ]) self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_141_get_dl_list_reinstall_notfound_fail(self, mock_query): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=TemplateVM state=Halted\n' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-name', + None)] = b'0\0test-vm' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-epoch', + None)] = b'0\x000' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-version', + None)] = b'0\x004.3' + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-release', + None)] = b'0\x0020200201' + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for test-vm\n' + ) + ] + args = argparse.Namespace(templates=['test-vm']) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.get_dl_list(args, self.app, + qubesadmin.tools.qvm_template.VersionSelector.REINSTALL) + self.assertTrue('Same version' in mock_err.getvalue()) + self.assertTrue('not found' in mock_err.getvalue()) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'qubes-template-test-vm') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_150_list_templates_installed_success(self, mock_query): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=TemplateVM state=Halted\n' \ + b'test-vm-2 class=TemplateVM state=Halted\n' \ + b'non-spec class=TemplateVM state=Halted\n' + build_time = '2020-09-01 14:30:00' # 1598970600 + install_time = '2020-09-01 15:30:00' + for key, val in [ + ('name', 'test-vm'), + ('epoch', '2'), + ('version', '4.1'), + ('release', '2020'), + ('reponame', '@commandline'), + ('buildtime', build_time), + ('installtime', install_time), + ('license', 'GPL'), + ('url', 'https://qubes-os.org'), + ('summary', 'Summary'), + ('description', 'Desc|desc')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + for key, val in [('name', 'test-vm-2-not-managed')]: + self.app.expected_calls[( + 'test-vm-2', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + for key, val in [ + ('name', 'non-spec'), + ('epoch', '0'), + ('version', '4.3'), + ('release', '20200201')]: + self.app.expected_calls[( + 'non-spec', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + args = argparse.Namespace( + all=False, + installed=True, + available=False, + extras=False, + upgrades=False, + machine_readable=False, + machine_readable_json=False, + templates=['test-vm*'] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out, \ + mock.patch.object(self.app.domains['test-vm'], + 'get_disk_utilization') as mock_disk: + mock_disk.return_value = 1234321 + qubesadmin.tools.qvm_template.list_templates( + args, self.app, 'list') + self.assertEqual(mock_out.getvalue(), +'''Installed Templates +test-vm 2:4.1-2020 @commandline +''') + self.assertEqual(mock_disk.mock_calls, [mock.call()]) + self.assertEqual(mock_query.mock_calls, []) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_151_list_templates_available_success(self, mock_query): + counter = 0 + def f(*args): + nonlocal counter + counter += 1 + if counter == 1: + return [ + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.2', + '20200201', + 'qubes-templates-itl-testing', + 2048576, + datetime.datetime(2020, 2, 23, 4, 56), + 'GPLv2', + 'https://qubes-os.org/?', + 'Qubes template for fedora-32 v2', + 'Qubes template\n for fedora-32 v2\n' + ) + ] + return [ + qubesadmin.tools.qvm_template.Template( + 'fedora-31', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ) + ] + mock_query.side_effect = f + args = argparse.Namespace( + all=False, + installed=False, + available=True, + extras=False, + upgrades=False, + machine_readable=False, + machine_readable_json=False, + templates=['fedora-32', 'fedora-31'] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out: + qubesadmin.tools.qvm_template.list_templates( + args, self.app, 'list') + # Order not determinstic because of sets + self.assertTrue(mock_out.getvalue() == \ +'''Available Templates +fedora-31 1:4.1-20200101 qubes-templates-itl +fedora-32 0:4.2-20200201 qubes-templates-itl-testing +''' \ + or mock_out.getvalue() == \ +'''Available Templates +fedora-32 0:4.2-20200201 qubes-templates-itl-testing +fedora-31 1:4.1-20200101 qubes-templates-itl +''') + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'fedora-32'), + mock.call(args, self.app, 'fedora-31') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_151_list_templates_available_all_success(self, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'fedora-31', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ) + ] + args = argparse.Namespace( + all=False, + installed=False, + available=True, + extras=False, + upgrades=False, + machine_readable=False, + machine_readable_json=False, + templates=[] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out: + qubesadmin.tools.qvm_template.list_templates( + args, self.app, 'list') + self.assertEqual(mock_out.getvalue(), +'''Available Templates +fedora-31 1:4.1-20200101 qubes-templates-itl +''') + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_152_list_templates_extras_success(self, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ) + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=TemplateVM state=Halted\n' \ + b'test-vm-2 class=TemplateVM state=Halted\n' \ + b'test-vm-3 class=TemplateVM state=Halted\n' \ + b'non-spec class=TemplateVM state=Halted\n' + for key, val in [('name', 'test-vm')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + for key, val in [ + ('name', 'test-vm-2'), + ('epoch', '1'), + ('version', '4.0'), + ('release', '2019'), + ('reponame', 'qubes-template-itl'), + ('buildtime', '2020-09-02 14:30:00'), + ('installtime', '2020-09-02 15:30:00'), + ('license', 'GPLv2'), + ('url', 'https://qubes-os.org/?'), + ('summary', 'Summary2'), + ('description', 'Desc|desc|2')]: + self.app.expected_calls[( + 'test-vm-2', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + for key, val in [('name', 'test-vm-3-non-managed')]: + self.app.expected_calls[( + 'test-vm-3', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + for key, val in [ + ('name', 'non-spec'), + ('epoch', '1'), + ('version', '4.0'), + ('release', '2019')]: + self.app.expected_calls[( + 'non-spec', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + args = argparse.Namespace( + all=False, + installed=False, + available=False, + extras=True, + upgrades=False, + machine_readable=False, + machine_readable_json=False, + templates=['test-vm*'] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out, \ + mock.patch.object(self.app.domains['test-vm-2'], + 'get_disk_utilization') as mock_disk: + mock_disk.return_value = 1234321 + qubesadmin.tools.qvm_template.list_templates( + args, self.app, 'list') + self.assertEqual(mock_out.getvalue(), +'''Extra Templates +test-vm-2 1:4.0-2019 qubes-template-itl +''') + self.assertEqual(mock_disk.mock_calls, [mock.call()]) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'test-vm*') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_153_list_templates_upgrades_success(self, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '0', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm-3', + '0', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ) + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=TemplateVM state=Halted\n' \ + b'test-vm-2 class=TemplateVM state=Halted\n' \ + b'test-vm-3 class=TemplateVM state=Halted\n' + for key, val in [ + ('name', 'test-vm'), + ('epoch', '1'), + ('version', '4.0'), + ('release', '2019')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + for key, val in [ + ('name', 'test-vm-2'), + ('epoch', '1'), + ('version', '4.0'), + ('release', '2019')]: + self.app.expected_calls[( + 'test-vm-2', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + for key, val in [('name', 'test-vm-3-non-managed')]: + self.app.expected_calls[( + 'test-vm-3', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + args = argparse.Namespace( + all=False, + installed=False, + available=False, + extras=False, + upgrades=True, + machine_readable=False, + machine_readable_json=False, + templates=['test-vm*'] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out, \ + mock.patch.object(self.app.domains['test-vm-2'], + 'get_disk_utilization') as mock_disk: + mock_disk.return_value = 1234321 + qubesadmin.tools.qvm_template.list_templates( + args, self.app, 'list') + self.assertEqual(mock_out.getvalue(), +'''Available Upgrades +test-vm 2:4.1-2020 qubes-templates-itl +''') + self.assertEqual(mock_disk.mock_calls, []) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'test-vm*') + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def __test_list_templates_all_success(self, operation, + args, expected, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ) + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm-2 class=TemplateVM state=Halted\n' + for key, val in [ + ('name', 'test-vm-2'), + ('epoch', '1'), + ('version', '4.0'), + ('release', '2019'), + ('reponame', '@commandline'), + ('buildtime', '2020-09-02 14:30:00'), + ('installtime', '2020-09-02 15:30:00'), + ('license', 'GPL'), + ('url', 'https://qubes-os.org'), + ('summary', 'Summary'), + ('description', 'Desc|desc')]: + self.app.expected_calls[( + 'test-vm-2', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out, \ + mock.patch.object(self.app.domains['test-vm-2'], + 'get_disk_utilization') as mock_disk: + mock_disk.return_value = 1234321 + qubesadmin.tools.qvm_template.list_templates( + args, self.app, operation) + self.assertEqual(mock_out.getvalue(), expected) + self.assertEqual(mock_disk.mock_calls, [mock.call()]) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app, 'test-vm*') + ]) + self.assertAllCalled() + + def test_154_list_templates_all_success(self): + args = argparse.Namespace( + all=True, + installed=False, + available=False, + extras=False, + upgrades=False, + machine_readable=False, + machine_readable_json=False, + templates=['test-vm*'] + ) + expected = \ +'''Installed Templates +test-vm-2 1:4.0-2019 @commandline +Available Templates +test-vm 2:4.1-2020 qubes-templates-itl +''' + self.__test_list_templates_all_success('list', args, expected) + + def test_155_list_templates_all_implicit_success(self): + args = argparse.Namespace( + all=False, + installed=False, + available=False, + extras=False, + upgrades=False, + machine_readable=False, + machine_readable_json=False, + templates=['test-vm*'] + ) + expected = \ +'''Installed Templates +test-vm-2 1:4.0-2019 @commandline +Available Templates +test-vm 2:4.1-2020 qubes-templates-itl +''' + self.__test_list_templates_all_success('list', args, expected) + + def test_156_list_templates_info_all_success(self): + args = argparse.Namespace( + all=False, + installed=False, + available=False, + extras=False, + upgrades=False, + machine_readable=False, + machine_readable_json=False, + templates=['test-vm*'] + ) + expected = \ +'''Installed Templates +Name : test-vm-2 +Epoch : 1 +Version : 4.0 +Release : 2019 +Size : 1.2 MiB +Repository : @commandline +Buildtime : 2020-09-02 14:30:00 +Install time : 2020-09-02 15:30:00 +URL : https://qubes-os.org +License : GPL +Summary : Summary +Description : Desc + : desc + +Available Templates +Name : test-vm +Epoch : 2 +Version : 4.1 +Release : 2020 +Size : 1.0 MiB +Repository : qubes-templates-itl +Buildtime : 2020-09-01 14:30:00+00:00 +URL : https://qubes-os.org +License : GPL +Summary : Qubes template for fedora-31 +Description : Qubes template + : for fedora-31 + +''' + self.__test_list_templates_all_success('info', args, expected) + + def test_157_list_templates_list_all_machinereadable_success(self): + args = argparse.Namespace( + all=False, + installed=False, + available=False, + extras=False, + upgrades=False, + machine_readable=True, + machine_readable_json=False, + templates=['test-vm*'] + ) + expected = \ +'''installed|test-vm-2|1:4.0-2019|@commandline +available|test-vm|2:4.1-2020|qubes-templates-itl +''' + self.__test_list_templates_all_success('list', args, expected) + + def test_158_list_templates_info_all_machinereadable_success(self): + args = argparse.Namespace( + all=False, + installed=False, + available=False, + extras=False, + upgrades=False, + machine_readable=True, + machine_readable_json=False, + templates=['test-vm*'] + ) + expected = \ +'''installed|test-vm-2|1|4.0|2019|@commandline|1234321|2020-09-02 14:30:00|2020-09-02 15:30:00|GPL|https://qubes-os.org|Summary|Desc|desc +available|test-vm|2|4.1|2020|qubes-templates-itl|1048576|2020-09-01 14:30:00||GPL|https://qubes-os.org|Qubes template for fedora-31|Qubes template| for fedora-31| +''' + self.__test_list_templates_all_success('info', args, expected) + + def test_159_list_templates_list_all_machinereadablejson_success(self): + args = argparse.Namespace( + all=False, + installed=False, + available=False, + extras=False, + upgrades=False, + machine_readable=False, + machine_readable_json=True, + templates=['test-vm*'] + ) + expected = \ +'''{"installed": [{"name": "test-vm-2", "evr": "1:4.0-2019", "reponame": "@commandline"}], "available": [{"name": "test-vm", "evr": "2:4.1-2020", "reponame": "qubes-templates-itl"}]} +''' + self.__test_list_templates_all_success('list', args, expected) + + def test_160_list_templates_info_all_machinereadablejson_success(self): + args = argparse.Namespace( + all=False, + installed=False, + available=False, + extras=False, + upgrades=False, + machine_readable=False, + machine_readable_json=True, + templates=['test-vm*'] + ) + expected = \ +r'''{"installed": [{"name": "test-vm-2", "epoch": "1", "version": "4.0", "release": "2019", "reponame": "@commandline", "size": "1234321", "buildtime": "2020-09-02 14:30:00", "installtime": "2020-09-02 15:30:00", "license": "GPL", "url": "https://qubes-os.org", "summary": "Summary", "description": "Desc\ndesc"}], "available": [{"name": "test-vm", "epoch": "2", "version": "4.1", "release": "2020", "reponame": "qubes-templates-itl", "size": "1048576", "buildtime": "2020-09-01 14:30:00", "installtime": "", "license": "GPL", "url": "https://qubes-os.org", "summary": "Qubes template for fedora-31", "description": "Qubes template\n for fedora-31\n"}]} +''' + self.__test_list_templates_all_success('info', args, expected) + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_161_list_templates_noresults_fail(self, mock_query): + mock_query.return_value = [] + args = argparse.Namespace( + all=False, + installed=False, + available=True, + extras=False, + upgrades=False, + machine_readable=False, + machine_readable_json=False, + templates=[] + ) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.list_templates( + args, self.app, 'list') + self.assertTrue('No matching templates' in mock_err.getvalue()) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_170_search_success(self, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '0', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Older Qubes template for fedora-31', + 'Older Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'should-not-match-3', + '0', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org/test-vm', + 'Qubes template for fedora-31', + 'test-vm Qubes template\n for fedora-31\n' + ) + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm-2 class=TemplateVM state=Halted\n' + for key, val in [ + ('name', 'test-vm-2'), + ('epoch', '1'), + ('version', '4.0'), + ('release', '2019'), + ('reponame', '@commandline'), + ('buildtime', '2020-09-02 14:30:00'), + ('license', 'GPL'), + ('url', 'https://qubes-os.org'), + ('summary', 'Summary'), + ('description', 'Desc|desc')]: + self.app.expected_calls[( + 'test-vm-2', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + args = argparse.Namespace( + all=False, + templates=['test-vm'] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out, \ + mock.patch.object(self.app.domains['test-vm-2'], + 'get_disk_utilization') as mock_disk: + mock_disk.return_value = 1234321 + qubesadmin.tools.qvm_template.search(args, self.app) + self.assertEqual(mock_out.getvalue(), +'''=== Name Exactly Matched: test-vm === +test-vm : Qubes template for fedora-31 +=== Name Matched: test-vm === +test-vm-2 : Summary +''') + self.assertEqual(mock_disk.mock_calls, [mock.call()]) + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_171_search_summary_success(self, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-template', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm :)', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-template-exact', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'test-vm', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm-2', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for test-vm-2', + 'Qubes template\n for fedora-31\n' + ), + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00' + args = argparse.Namespace( + all=False, + templates=['test-vm'] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out: + qubesadmin.tools.qvm_template.search(args, self.app) + self.assertEqual(mock_out.getvalue(), +'''=== Name & Summary Matched: test-vm === +test-vm : Qubes template for test-vm +test-vm-2 : Qubes template for test-vm-2 +=== Summary Matched: test-vm === +test-template : Qubes template for test-vm :) +=== Summary Exactly Matched: test-vm === +test-template-exact : test-vm +''') + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_172_search_namesummaryexact_success(self, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-template-exact', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'test-vm', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'test-vm', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm-2', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'test-vm', + 'Qubes template\n for fedora-31\n' + ) + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00' + args = argparse.Namespace( + all=False, + templates=['test-vm'] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out: + qubesadmin.tools.qvm_template.search(args, self.app) + self.assertEqual(mock_out.getvalue(), +'''=== Name & Summary Exactly Matched: test-vm === +test-vm : test-vm +=== Name & Summary Matched: test-vm === +test-vm-2 : test-vm +=== Summary Exactly Matched: test-vm === +test-template-exact : test-vm +''') + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_173_search_multiquery_success(self, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-template-exact', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'test-vm', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'test-vm', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'should-not-match', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Summary', + 'test-vm Qubes template\n for fedora-31\n' + ) + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00' + args = argparse.Namespace( + all=False, + templates=['test-vm', 'test-template'] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out: + qubesadmin.tools.qvm_template.search(args, self.app) + self.assertEqual(mock_out.getvalue(), +'''=== Name & Summary Matched: test-template, test-vm === +test-template-exact : test-vm +''') + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_174_search_multiquery_exact_success(self, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'summary', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'summary', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'test-vm Summary', + 'Qubes template\n for fedora-31\n' + ) + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00' + args = argparse.Namespace( + all=False, + templates=['test-vm', 'summary'] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out: + qubesadmin.tools.qvm_template.search(args, self.app) + self.assertEqual(mock_out.getvalue(), +'''=== Name & Summary Matched: summary, test-vm === +summary : test-vm Summary +=== Name & Summary Exactly Matched: summary, test-vm === +test-vm : summary +''') + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_175_search_all_success(self, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org/keyword-url', + 'summary', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm-exact', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'test-vm Summary', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm-exac2', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'test-vm-exac2', + 'test-vm Summary', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'test-vm-2', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'test-vm Summary', + 'keyword-desc' + ), + qubesadmin.tools.qvm_template.Template( + 'should-not-match', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Summary', + 'Description' + ) + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00' + args = argparse.Namespace( + all=True, + templates=['test-vm-exact', 'test-vm-exac2', + 'keyword-url', 'keyword-desc'] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out: + qubesadmin.tools.qvm_template.search(args, self.app) + self.assertEqual(mock_out.getvalue(), +'''=== Name & URL Exactly Matched: test-vm-exac2 === +test-vm-exac2 : test-vm Summary +=== Name Exactly Matched: test-vm-exact === +test-vm-exact : test-vm Summary +=== Description Exactly Matched: keyword-desc === +test-vm-2 : test-vm Summary +=== URL Matched: keyword-url === +test-vm : summary +''') + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_176_search_wildcard_success(self, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'test-vm', + '2', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'should-not-match-3', + '0', + '4.1', + '2020', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 9, 1, 14, 30, + tzinfo=datetime.timezone.utc), + 'GPL', + 'https://qubes-os.org/test-vm', + 'Qubes template for fedora-31', + 'test-vm Qubes template\n for fedora-31\n' + ) + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00' + args = argparse.Namespace( + all=False, + templates=['t?st-vm'] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out: + qubesadmin.tools.qvm_template.search(args, self.app) + self.assertEqual(mock_out.getvalue(), +'''=== Name Matched: t?st-vm === +test-vm : Qubes template for fedora-31 +''') + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app) + ]) + self.assertAllCalled() From 63f488f64cd65f68c4bff42658452c6c890d8694 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 15 Sep 2020 01:09:45 +0800 Subject: [PATCH 078/119] qvm-template: Mock print_table in tests for consistent output --- qubesadmin/tests/tools/qvm_template.py | 69 ++++++++++---------------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index f2ea27b..6713c5f 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -15,10 +15,18 @@ import qubesadmin.tools.qvm_template class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): def setUp(self): - self.maxDiff = 1e9 + # Print str(list) directly so that the output is consistent no matter + # which implementation of `column` we use + self.mock_table = mock.patch('qubesadmin.tools.print_table') + mock_table = self.mock_table.start() + def print_table(table, *args): + print(str(table)) + mock_table.side_effect = print_table + super().setUp() def tearDown(self): + self.mock_table.stop() super().tearDown() def test_000_verify_rpm_success(self): @@ -2239,7 +2247,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 args, self.app, 'list') self.assertEqual(mock_out.getvalue(), '''Installed Templates -test-vm 2:4.1-2020 @commandline +[('test-vm', '2:4.1-2020', '@commandline')] ''') self.assertEqual(mock_disk.mock_calls, [mock.call()]) self.assertEqual(mock_query.mock_calls, []) @@ -2297,15 +2305,17 @@ test-vm 2:4.1-2020 @commandline qubesadmin.tools.qvm_template.list_templates( args, self.app, 'list') # Order not determinstic because of sets + expected = [ + ('fedora-31', '1:4.1-20200101', 'qubes-templates-itl'), + ('fedora-32', '0:4.2-20200201', 'qubes-templates-itl-testing') + ] self.assertTrue(mock_out.getvalue() == \ -'''Available Templates -fedora-31 1:4.1-20200101 qubes-templates-itl -fedora-32 0:4.2-20200201 qubes-templates-itl-testing +f'''Available Templates +{str([expected[1], expected[0]])} ''' \ or mock_out.getvalue() == \ -'''Available Templates -fedora-32 0:4.2-20200201 qubes-templates-itl-testing -fedora-31 1:4.1-20200101 qubes-templates-itl +f'''Available Templates +{str([expected[0], expected[1]])} ''') self.assertEqual(mock_query.mock_calls, [ mock.call(args, self.app, 'fedora-32'), @@ -2345,7 +2355,7 @@ fedora-31 1:4.1-20200101 qubes-templates-itl args, self.app, 'list') self.assertEqual(mock_out.getvalue(), '''Available Templates -fedora-31 1:4.1-20200101 qubes-templates-itl +[('fedora-31', '1:4.1-20200101', 'qubes-templates-itl')] ''') self.assertEqual(mock_query.mock_calls, [ mock.call(args, self.app) @@ -2432,7 +2442,7 @@ fedora-31 1:4.1-20200101 qubes-templates-itl args, self.app, 'list') self.assertEqual(mock_out.getvalue(), '''Extra Templates -test-vm-2 1:4.0-2019 qubes-template-itl +[('test-vm-2', '1:4.0-2019', 'qubes-template-itl')] ''') self.assertEqual(mock_disk.mock_calls, [mock.call()]) self.assertEqual(mock_query.mock_calls, [ @@ -2534,7 +2544,7 @@ test-vm-2 1:4.0-2019 qubes-template-itl args, self.app, 'list') self.assertEqual(mock_out.getvalue(), '''Available Upgrades -test-vm 2:4.1-2020 qubes-templates-itl +[('test-vm', '2:4.1-2020', 'qubes-templates-itl')] ''') self.assertEqual(mock_disk.mock_calls, []) self.assertEqual(mock_query.mock_calls, [ @@ -2606,9 +2616,9 @@ test-vm 2:4.1-2020 qubes-templates-itl ) expected = \ '''Installed Templates -test-vm-2 1:4.0-2019 @commandline +[('test-vm-2', '1:4.0-2019', '@commandline')] Available Templates -test-vm 2:4.1-2020 qubes-templates-itl +[('test-vm', '2:4.1-2020', 'qubes-templates-itl')] ''' self.__test_list_templates_all_success('list', args, expected) @@ -2625,9 +2635,9 @@ test-vm 2:4.1-2020 qubes-templates-itl ) expected = \ '''Installed Templates -test-vm-2 1:4.0-2019 @commandline +[('test-vm-2', '1:4.0-2019', '@commandline')] Available Templates -test-vm 2:4.1-2020 qubes-templates-itl +[('test-vm', '2:4.1-2020', 'qubes-templates-itl')] ''' self.__test_list_templates_all_success('list', args, expected) @@ -2644,34 +2654,9 @@ test-vm 2:4.1-2020 qubes-templates-itl ) expected = \ '''Installed Templates -Name : test-vm-2 -Epoch : 1 -Version : 4.0 -Release : 2019 -Size : 1.2 MiB -Repository : @commandline -Buildtime : 2020-09-02 14:30:00 -Install time : 2020-09-02 15:30:00 -URL : https://qubes-os.org -License : GPL -Summary : Summary -Description : Desc - : desc - +[('Name', ':', 'test-vm-2'), ('Epoch', ':', '1'), ('Version', ':', '4.0'), ('Release', ':', '2019'), ('Size', ':', '1.2 MiB'), ('Repository', ':', '@commandline'), ('Buildtime', ':', '2020-09-02 14:30:00'), ('Install time', ':', '2020-09-02 15:30:00'), ('URL', ':', 'https://qubes-os.org'), ('License', ':', 'GPL'), ('Summary', ':', 'Summary'), ('Description', ':', 'Desc'), ('', ':', 'desc'), (' ', ' ', ' ')] Available Templates -Name : test-vm -Epoch : 2 -Version : 4.1 -Release : 2020 -Size : 1.0 MiB -Repository : qubes-templates-itl -Buildtime : 2020-09-01 14:30:00+00:00 -URL : https://qubes-os.org -License : GPL -Summary : Qubes template for fedora-31 -Description : Qubes template - : for fedora-31 - +[('Name', ':', 'test-vm'), ('Epoch', ':', '2'), ('Version', ':', '4.1'), ('Release', ':', '2020'), ('Size', ':', '1.0 MiB'), ('Repository', ':', 'qubes-templates-itl'), ('Buildtime', ':', '2020-09-01 14:30:00+00:00'), ('URL', ':', 'https://qubes-os.org'), ('License', ':', 'GPL'), ('Summary', ':', 'Qubes template for fedora-31'), ('Description', ':', 'Qubes template'), ('', ':', ' for fedora-31'), (' ', ' ', ' ')] ''' self.__test_list_templates_all_success('info', args, expected) From 5f0364046786247736abd6d0f57ae1a7ba7602b0 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 15 Sep 2020 01:30:57 +0800 Subject: [PATCH 079/119] qvm-template: Fix broken indention --- qubesadmin/tools/qvm_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 99d381b..4227975 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -1067,7 +1067,7 @@ def list_templates(args: argparse.Namespace, for vm in app.domains: if is_managed_template(vm) and \ check_append(vm.name, query_local_evr(vm)): - append_vm(vm, TemplateState.INSTALLED) + append_vm(vm, TemplateState.INSTALLED) if args.available or args.all: # Spec should already be checked by repoquery From 1671b4216fe61e46dc0022a0a147c76fbdfd9d51 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 4 Oct 2020 03:05:14 +0800 Subject: [PATCH 080/119] qvm-template: Add tests for download function and fix minor bugs --- qubesadmin/tests/tools/qvm_template.py | 289 ++++++++++++++++++++++++- qubesadmin/tools/qvm_template.py | 26 ++- 2 files changed, 301 insertions(+), 14 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 6713c5f..8638f8e 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -191,7 +191,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_dl_list.return_value = {} mock_call.side_effect = self.add_new_vm_side_effect mock_time = mock.Mock(wraps=datetime.datetime) - mock_time.today.return_value = \ + mock_time.now.return_value = \ datetime.datetime(2020, 9, 1, 15, 30, tzinfo=datetime.timezone.utc) with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ mock.patch('datetime.datetime', new=mock_time), \ @@ -322,7 +322,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_dl_list.return_value = {} mock_call.side_effect = self.add_new_vm_side_effect mock_time = mock.Mock(wraps=datetime.datetime) - mock_time.today.return_value = \ + mock_time.now.return_value = \ datetime.datetime(2020, 9, 1, 15, 30, tzinfo=datetime.timezone.utc) with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ mock.patch('datetime.datetime', new=mock_time), \ @@ -3243,3 +3243,288 @@ test-vm : Qubes template for fedora-31 mock.call(args, self.app) ]) self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_180_download_success(self, mock_qrexec, mock_dllist): + with tempfile.TemporaryDirectory() as dir: + args = argparse.Namespace( + retries=1 + ) + qubesadmin.tools.qvm_template.download(args, self.app, dir, { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576), + 'fedora-32': qubesadmin.tools.qvm_template.DlEntry( + ('0', '1', '2'), + 'qubes-templates-itl-testing', + 2048576) + }, '.unverified') + self.assertEqual(mock_qrexec.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', + dir + '/qubes-template-fedora-31-1:2-3.rpm.unverified', + 1048576), + mock.call(args, self.app, 'qubes-template-fedora-32-0:1-2', + dir + '/qubes-template-fedora-32-0:1-2.rpm.unverified', + 2048576) + ]) + self.assertEqual(mock_dllist.mock_calls, []) + self.assertTrue(all( + [x.endswith('.unverified') for x in os.listdir(dir)])) + + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_181_download_success_nosuffix(self, mock_qrexec, mock_dllist): + with tempfile.TemporaryDirectory() as dir: + args = argparse.Namespace( + retries=1, + downloaddir=dir + ) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + qubesadmin.tools.qvm_template.download(args, self.app, None, { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576) + }) + self.assertEqual(mock_qrexec.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', + dir + '/qubes-template-fedora-31-1:2-3.rpm', + 1048576) + ]) + self.assertEqual(mock_dllist.mock_calls, []) + + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_182_download_success_getdllist(self, mock_qrexec, mock_dllist): + mock_dllist.return_value = { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576) + } + with tempfile.TemporaryDirectory() as dir: + args = argparse.Namespace( + retries=1 + ) + qubesadmin.tools.qvm_template.download(args, self.app, + dir, None, '.unverified', + qubesadmin.tools.qvm_template.VersionSelector.LATEST_LOWER) + self.assertEqual(mock_qrexec.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', + dir + '/qubes-template-fedora-31-1:2-3.rpm.unverified', + 1048576) + ]) + self.assertEqual(mock_dllist.mock_calls, [ + mock.call(args, self.app, + version_selector=\ + qubesadmin.tools.qvm_template.\ + VersionSelector.LATEST_LOWER) + ]) + self.assertTrue(all( + [x.endswith('.unverified') for x in os.listdir(dir)])) + + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_183_download_success_downloaddir(self, mock_qrexec, mock_dllist): + with tempfile.TemporaryDirectory() as dir: + args = argparse.Namespace( + retries=1, + downloaddir=dir + ) + qubesadmin.tools.qvm_template.download(args, self.app, None, { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576) + }, '.unverified') + self.assertEqual(mock_qrexec.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', + dir + '/qubes-template-fedora-31-1:2-3.rpm.unverified', + 1048576) + ]) + self.assertEqual(mock_dllist.mock_calls, []) + self.assertTrue(all( + [x.endswith('.unverified') for x in os.listdir(dir)])) + + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_184_download_success_exists(self, mock_qrexec, mock_dllist): + with tempfile.TemporaryDirectory() as dir: + with open(os.path.join( + dir, 'qubes-template-fedora-31-1:2-3.rpm.unverified'), + 'w') as _: + pass + args = argparse.Namespace( + retries=1, + downloaddir=dir + ) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + qubesadmin.tools.qvm_template.download(args, self.app, None, { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576), + 'fedora-32': qubesadmin.tools.qvm_template.DlEntry( + ('0', '1', '2'), + 'qubes-templates-itl-testing', + 2048576) + }, '.unverified') + self.assertTrue('already exists, skipping' + in mock_err.getvalue()) + self.assertEqual(mock_qrexec.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-32-0:1-2', + dir + '/qubes-template-fedora-32-0:1-2.rpm.unverified', + 2048576) + ]) + self.assertEqual(mock_dllist.mock_calls, []) + self.assertTrue(all( + [x.endswith('.unverified') for x in os.listdir(dir)])) + + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_185_download_success_existsmove(self, mock_qrexec, mock_dllist): + with tempfile.TemporaryDirectory() as dir: + with open(os.path.join( + dir, 'qubes-template-fedora-31-1:2-3.rpm'), + 'w') as _: + pass + args = argparse.Namespace( + retries=1, + downloaddir=dir + ) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + qubesadmin.tools.qvm_template.download(args, self.app, None, { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576) + }, '.unverified') + self.assertTrue('already exists, skipping' + in mock_err.getvalue()) + self.assertEqual(mock_qrexec.mock_calls, []) + self.assertEqual(mock_dllist.mock_calls, []) + self.assertTrue(os.path.exists( + dir + '/qubes-template-fedora-31-1:2-3.rpm.unverified')) + self.assertTrue(all( + [x.endswith('.unverified') for x in os.listdir(dir)])) + + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_186_download_success_existsnosuffix(self, mock_qrexec, mock_dllist): + with tempfile.TemporaryDirectory() as dir: + with open(os.path.join( + dir, 'qubes-template-fedora-31-1:2-3.rpm'), + 'w') as _: + pass + args = argparse.Namespace( + retries=1, + downloaddir=dir + ) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + qubesadmin.tools.qvm_template.download(args, self.app, None, { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576) + }) + self.assertTrue('already exists, skipping' + in mock_err.getvalue()) + self.assertEqual(mock_qrexec.mock_calls, []) + self.assertEqual(mock_dllist.mock_calls, []) + self.assertTrue(os.path.exists( + dir + '/qubes-template-fedora-31-1:2-3.rpm')) + + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_187_download_success_retry(self, mock_qrexec, mock_dllist): + counter = 0 + def f(*args): + nonlocal counter + counter += 1 + if counter == 1: + raise ConnectionError + mock_qrexec.side_effect = f + with tempfile.TemporaryDirectory() as dir: + args = argparse.Namespace( + retries=2, + downloaddir=dir + ) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ + mock.patch('os.remove') as mock_rm: + qubesadmin.tools.qvm_template.download(args, self.app, None, { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576) + }) + self.assertTrue('retrying...' in mock_err.getvalue()) + self.assertEqual(mock_rm.mock_calls, [ + mock.call(dir + '/qubes-template-fedora-31-1:2-3.rpm') + ]) + self.assertEqual(mock_qrexec.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', + dir + '/qubes-template-fedora-31-1:2-3.rpm', + 1048576), + mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', + dir + '/qubes-template-fedora-31-1:2-3.rpm', + 1048576) + ]) + self.assertEqual(mock_dllist.mock_calls, []) + + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_188_download_fail_retry(self, mock_qrexec, mock_dllist): + counter = 0 + def f(*args): + nonlocal counter + counter += 1 + if counter <= 3: + raise ConnectionError + mock_qrexec.side_effect = f + with tempfile.TemporaryDirectory() as dir: + args = argparse.Namespace( + retries=3, + downloaddir=dir + ) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ + mock.patch('os.remove') as mock_rm: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.download( + args, self.app, None, { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576) + }) + self.assertEqual(mock_err.getvalue().count('retrying...'), 2) + self.assertTrue('download failed' in mock_err.getvalue()) + self.assertEqual(mock_rm.mock_calls, [ + mock.call(dir + '/qubes-template-fedora-31-1:2-3.rpm'), + mock.call(dir + '/qubes-template-fedora-31-1:2-3.rpm'), + mock.call(dir + '/qubes-template-fedora-31-1:2-3.rpm') + ]) + self.assertEqual(mock_qrexec.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', + dir + '/qubes-template-fedora-31-1:2-3.rpm', + 1048576), + mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', + dir + '/qubes-template-fedora-31-1:2-3.rpm', + 1048576), + mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', + dir + '/qubes-template-fedora-31-1:2-3.rpm', + 1048576) + ]) + self.assertEqual(mock_dllist.mock_calls, []) + + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_189_download_fail_interrupt(self, mock_qrexec, mock_dllist): + def f(*args): + raise RuntimeError + mock_qrexec.side_effect = f + with tempfile.TemporaryDirectory() as dir: + args = argparse.Namespace( + retries=3, + downloaddir=dir + ) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ + mock.patch('os.remove') as mock_rm: + with self.assertRaises(RuntimeError): + qubesadmin.tools.qvm_template.download( + args, self.app, None, { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576) + }) + self.assertEqual(mock_rm.mock_calls, [ + mock.call(dir + '/qubes-template-fedora-31-1:2-3.rpm') + ]) + self.assertEqual(mock_qrexec.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', + dir + '/qubes-template-fedora-31-1:2-3.rpm', + 1048576) + ]) + self.assertEqual(mock_dllist.mock_calls, []) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 4227975..dfc006b 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -53,6 +53,8 @@ def qubes_release() -> str: continue val = val.strip('\'"') # strip possible quotes return val + # Return default value instead of throwing so that it works on CI + return '4.1' def parser_gen() -> argparse.ArgumentParser: """Generate argument parser for the application.""" @@ -257,7 +259,7 @@ def is_match_spec(name: str, epoch: str, version: str, release: str, spec: str :return: A tuple. The first element indicates whether there is a match; the second element represents the priority of the match (lower is better) """ - if epoch != 0: + if epoch != '0': targets = [ f'{name}-{epoch}:{version}-{release}', f'{name}', @@ -713,14 +715,13 @@ def download( spec = PACKAGE_NAME_PREFIX + name + '-' + version_str target = os.path.join(path, '%s.rpm' % spec) target_suffix = target + suffix - if suffix != '' and os.path.exists(target_suffix): + if os.path.exists(target_suffix): print('\'%s\' already exists, skipping...' % target, file=sys.stderr) - if os.path.exists(target): + elif os.path.exists(target): print('\'%s\' already exists, skipping...' % target, file=sys.stderr) - if suffix != '': - os.rename(target, target_suffix) + os.rename(target, target_suffix) else: print('Downloading \'%s\'...' % spec, file=sys.stderr) done = False @@ -930,7 +931,7 @@ def install( tz=datetime.timezone.utc) \ .strftime(DATE_FMT) tpl.features['template-installtime'] = \ - datetime.datetime.today( + datetime.datetime.now( tz=datetime.timezone.utc).strftime(DATE_FMT) tpl.features['template-license'] = \ package_hdr[rpm.RPMTAG_LICENSE] @@ -1100,18 +1101,19 @@ def list_templates(args: argparse.Namespace, if args.machine_readable: if operation == 'info': - tpl_list = info_to_machine_output(tpl_list) + tpl_list_dict = info_to_machine_output(tpl_list) elif operation == 'list': - tpl_list = list_to_machine_output(tpl_list) - for status, grp in tpl_list.items(): + tpl_list_dict = list_to_machine_output(tpl_list) + for status, grp in tpl_list_dict.items(): for line in grp: print('|'.join([status] + list(line.values()))) elif args.machine_readable_json: if operation == 'info': - tpl_list = info_to_machine_output(tpl_list, replace_newline=False) + tpl_list_dict = \ + info_to_machine_output(tpl_list, replace_newline=False) elif operation == 'list': - tpl_list = list_to_machine_output(tpl_list) - print(json.dumps(tpl_list)) + tpl_list_dict = list_to_machine_output(tpl_list) + print(json.dumps(tpl_list_dict)) else: if operation == 'info': tpl_list = info_to_human_output(tpl_list) From b500462abbe4f8818a176e8cce555907006333dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Jan 2021 22:42:57 +0100 Subject: [PATCH 081/119] qvm-template: use UpdateVM by default Do not hardcode sys-firewall --- doc/manpages/qvm-template.rst | 2 +- qubesadmin/tools/qvm_template.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/manpages/qvm-template.rst b/doc/manpages/qvm-template.rst index 406f237..1bc179d 100644 --- a/doc/manpages/qvm-template.rst +++ b/doc/manpages/qvm-template.rst @@ -31,7 +31,7 @@ Options .. option:: --updatevm UPDATEVM Specify VM to download updates from. (Set to empty string to specify the - current VM.) (default: sys-firewall) + current VM.) (default: same as UpdateVM - see ``qubes-prefs``) .. option:: --enablerepo REPOID diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index dfc006b..05f74f9 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -37,6 +37,8 @@ UNVERIFIED_SUFFIX = '.unverified' LOCK_FILE = '/var/tmp/qvm-template.lck' DATE_FMT = '%Y-%m-%d %H:%M:%S' +UPDATEVM = str('global UpdateVM') + def qubes_release() -> str: """Return the Qubes release.""" if os.path.exists('/usr/share/qubes/marker-vm'): @@ -78,7 +80,7 @@ def parser_gen() -> argparse.ArgumentParser: parser_main.add_argument('--keyring', default='/usr/share/qubes/repo-templates/keys', help='Specify directory containing RPM public keys.') - parser_main.add_argument('--updatevm', default='sys-firewall', + parser_main.add_argument('--updatevm', default=UPDATEVM, help=('Specify VM to download updates from.' ' (Set to empty string to specify the current VM.)')) parser_main.add_argument('--enablerepo', action='append', default=[], @@ -1384,6 +1386,9 @@ def main(args: typing.Optional[typing.Sequence[str]] = None, if app is None: app = qubesadmin.Qubes() + if p_args.updatevm is UPDATEVM: + p_args.updatevm = app.updatevm + if p_args.refresh: qrexec_repoquery(p_args, app, refresh=True) From f3f6750a3ff8b114b2a598dc7b76354b1daa1b76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Jan 2021 22:47:38 +0100 Subject: [PATCH 082/119] qvm-template: call rpmkeys --checksig for signature verification RPM API is confusing and it's easy to get it wrong when verifying package signatures. Call 'rpmkeys --checksig' which is more rebust here - RPM authors should know how to use their API. QubesOS/qubes-issues#2534 --- qubesadmin/tests/tools/qvm_template.py | 143 +++++++++---------------- qubesadmin/tools/qvm_template.py | 57 +++++----- test-packages/rpm.py | 14 +-- 3 files changed, 91 insertions(+), 123 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 8638f8e..a150c1b 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -29,53 +29,75 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): self.mock_table.stop() super().tearDown() - def test_000_verify_rpm_success(self): - ts = mock.MagicMock() + @mock.patch('rpm.TransactionSet') + @mock.patch('subprocess.check_call') + @mock.patch('subprocess.check_output') + def test_000_verify_rpm_success(self, mock_proc, mock_call, mock_ts): # Just return a dict instead of rpm.hdr hdr = { rpm.RPMTAG_SIGPGP: 'xxx', # non-empty rpm.RPMTAG_SIGGPG: 'xxx', # non-empty } - ts.hdrFromFdno.return_value = hdr - ret = qubesadmin.tools.qvm_template.verify_rpm('/dev/null', ts) - ts.hdrFromFdno.assert_called_once() + mock_ts.return_value.hdrFromFdno.return_value = hdr + mock_proc.return_value = b'dummy.rpm: digests signatures OK\n' + ret = qubesadmin.tools.qvm_template.verify_rpm('/dev/null', + ['/path/to/key']) + mock_call.assert_called_once() + mock_proc.assert_called_once() self.assertEqual(hdr, ret) self.assertAllCalled() - def test_001_verify_rpm_nosig_fail(self): - ts = mock.MagicMock() + @mock.patch('rpm.TransactionSet') + @mock.patch('subprocess.check_call') + @mock.patch('subprocess.check_output') + def test_001_verify_rpm_nosig_fail(self, mock_proc, mock_call, mock_ts): # Just return a dict instead of rpm.hdr hdr = { rpm.RPMTAG_SIGPGP: None, # empty rpm.RPMTAG_SIGGPG: None, # empty } - ts.hdrFromFdno.return_value = hdr - ret = qubesadmin.tools.qvm_template.verify_rpm('/dev/null', ts) - ts.hdrFromFdno.assert_called_once() - self.assertEqual(ret, None) + mock_ts.return_value.hdrFromFdno.return_value = hdr + mock_proc.return_value = b'dummy.rpm: digests OK\n' + with self.assertRaises(Exception) as e: + qubesadmin.tools.qvm_template.verify_rpm('/dev/null', + ['/path/to/key']) + mock_call.assert_called_once() + mock_proc.assert_called_once() + self.assertIn('Signature verification failed', e.exception.args[0]) + mock_ts.assert_not_called() self.assertAllCalled() - def test_002_verify_rpm_nosig_success(self): - ts = mock.MagicMock() + @mock.patch('rpm.TransactionSet') + @mock.patch('subprocess.check_call') + @mock.patch('subprocess.check_output') + def test_002_verify_rpm_nosig_success(self, mock_proc, mock_call, mock_ts): # Just return a dict instead of rpm.hdr hdr = { rpm.RPMTAG_SIGPGP: None, # empty rpm.RPMTAG_SIGGPG: None, # empty } - ts.hdrFromFdno.return_value = hdr - ret = qubesadmin.tools.qvm_template.verify_rpm('/dev/null', ts, True) - ts.hdrFromFdno.assert_called_once() + mock_ts.return_value.hdrFromFdno.return_value = hdr + mock_proc.return_value = b'dummy.rpm: digests OK\n' + ret = qubesadmin.tools.qvm_template.verify_rpm('/dev/null', + ['/path/to/key'], True) + mock_proc.assert_not_called() + mock_call.assert_not_called() self.assertEqual(ret, hdr) self.assertAllCalled() - def test_003_verify_rpm_badsig_fail(self): - ts = mock.MagicMock() - def f(*args): - raise rpm.error('public key not trusted') - ts.hdrFromFdno.side_effect = f - ret = qubesadmin.tools.qvm_template.verify_rpm('/dev/null', ts) - ts.hdrFromFdno.assert_called_once() - self.assertEqual(ret, None) + @mock.patch('rpm.TransactionSet') + @mock.patch('subprocess.check_call') + @mock.patch('subprocess.check_output') + def test_003_verify_rpm_badsig_fail(self, mock_proc, mock_call, mock_ts): + mock_proc.side_effect = subprocess.CalledProcessError(1, + ['rpmkeys', '--checksig'], b'/dev/null: digests SIGNATURES NOT OK\n') + with self.assertRaises(Exception) as e: + qubesadmin.tools.qvm_template.verify_rpm('/dev/null', + ['/path/to/key']) + mock_call.assert_called_once() + mock_proc.assert_called_once() + self.assertIn('Signature verification failed', e.exception.args[0]) + mock_ts.assert_not_called() self.assertAllCalled() @mock.patch('subprocess.Popen') @@ -144,10 +166,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): @mock.patch('qubesadmin.tools.qvm_template.download') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') - @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') def test_100_install_local_success( self, - mock_ts, mock_verify, mock_dl_list, mock_dl, @@ -201,7 +221,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): path = template_file.name args = argparse.Namespace( templates=[path], - keyring='/usr/share/qubes/repo-templates/keys', + keyring='/tmp', nogpgcheck=False, cachedir='/var/cache/qvm-template', yes=False, @@ -217,15 +237,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock.call().__enter__(), mock.call().__exit__(None, None, None) ]) - # Keyring created - self.assertEqual(mock_ts.mock_calls, [ - mock.call('/usr/share/qubes/repo-templates/keys') - ]) - # Package verified - self.assertEqual(mock_verify.mock_calls, [ - mock.call(path, mock_ts('/usr/share/qubes/repo-templates/keys'), - False) - ]) # Attempt to get download list selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST self.assertEqual(mock_dl_list.mock_calls, [ @@ -275,10 +286,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): @mock.patch('qubesadmin.tools.qvm_template.download') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') - @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') def test_101_install_local_postprocargs_success( self, - mock_ts, mock_verify, mock_dl_list, mock_dl, @@ -332,7 +341,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): path = template_file.name args = argparse.Namespace( templates=[path], - keyring='/usr/share/qubes/repo-templates/keys', + keyring='/tmp', nogpgcheck=False, cachedir='/var/cache/qvm-template', yes=False, @@ -348,15 +357,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock.call().__enter__(), mock.call().__exit__(None, None, None) ]) - # Keyring created - self.assertEqual(mock_ts.mock_calls, [ - mock.call('/usr/share/qubes/repo-templates/keys') - ]) - # Package verified - self.assertEqual(mock_verify.mock_calls, [ - mock.call(path, mock_ts('/usr/share/qubes/repo-templates/keys'), - False) - ]) # Attempt to get download list selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST self.assertEqual(mock_dl_list.mock_calls, [ @@ -409,10 +409,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): @mock.patch('qubesadmin.tools.qvm_template.download') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') - @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') def test_102_install_local_badsig_fail( self, - mock_ts, mock_verify, mock_dl_list, mock_dl, @@ -432,7 +430,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): path = template_file.name args = argparse.Namespace( templates=[path], - keyring='/usr/share/qubes/repo-templates/keys', + keyring='/tmp', nogpgcheck=False, cachedir='/var/cache/qvm-template', yes=False, @@ -452,15 +450,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # Check error message self.assertTrue('verification failed' in mock_err.getvalue()) - # Keyring created - self.assertEqual(mock_ts.mock_calls, [ - mock.call('/usr/share/qubes/repo-templates/keys') - ]) - # Package verified - self.assertEqual(mock_verify.mock_calls, [ - mock.call(path, mock_ts('/usr/share/qubes/repo-templates/keys'), - False) - ]) # Should not be executed: self.assertEqual(mock_dl_list.mock_calls, []) self.assertEqual(mock_dl.mock_calls, []) @@ -483,10 +472,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): @mock.patch('qubesadmin.tools.qvm_template.download') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') - @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') def test_103_install_local_exists_fail( self, - mock_ts, mock_verify, mock_dl_list, mock_dl, @@ -519,7 +506,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): path = template_file.name args = argparse.Namespace( templates=[path], - keyring='/usr/share/qubes/repo-templates/keys', + keyring='/tmp', nogpgcheck=False, cachedir='/var/cache/qvm-template', yes=False, @@ -537,15 +524,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # Check warning message self.assertTrue('already installed' in mock_err.getvalue()) - # Keyring created - self.assertEqual(mock_ts.mock_calls, [ - mock.call('/usr/share/qubes/repo-templates/keys') - ]) - # Package verified - self.assertEqual(mock_verify.mock_calls, [ - mock.call(path, mock_ts('/usr/share/qubes/repo-templates/keys'), - False) - ]) # Attempt to get download list selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST self.assertEqual(mock_dl_list.mock_calls, [ @@ -576,10 +554,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): @mock.patch('qubesadmin.tools.qvm_template.download') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') - @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') def test_104_install_local_badpkgname_fail( self, - mock_ts, mock_verify, mock_dl_list, mock_dl, @@ -609,7 +585,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): path = template_file.name args = argparse.Namespace( templates=[path], - keyring='/usr/share/qubes/repo-templates/keys', + keyring='/tmp', nogpgcheck=False, cachedir='/var/cache/qvm-template', yes=False, @@ -628,15 +604,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # Check error message self.assertTrue('Illegal package name' in mock_err.getvalue()) - # Keyring created - self.assertEqual(mock_ts.mock_calls, [ - mock.call('/usr/share/qubes/repo-templates/keys') - ]) - # Package verified - self.assertEqual(mock_verify.mock_calls, [ - mock.call(path, mock_ts('/usr/share/qubes/repo-templates/keys'), - False) - ]) # Should not be executed: self.assertEqual(mock_dl_list.mock_calls, []) self.assertEqual(mock_dl.mock_calls, []) @@ -720,10 +687,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): @mock.patch('qubesadmin.tools.qvm_template.download') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') - @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') def test_106_install_local_badpath_fail( self, - mock_ts, mock_verify, mock_dl_list, mock_dl, @@ -741,7 +706,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): path = '/var/tmp/ShOulD-NoT-ExIsT.rpm' args = argparse.Namespace( templates=[path], - keyring='/usr/share/qubes/repo-templates/keys', + keyring='/tmp', nogpgcheck=False, cachedir='/var/cache/qvm-template', yes=False, @@ -761,10 +726,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): # Check error message self.assertTrue(f"RPM file '{path}' not found" \ in mock_err.getvalue()) - # Keyring created - self.assertEqual(mock_ts.mock_calls, [ - mock.call('/usr/share/qubes/repo-templates/keys') - ]) # Should not be executed: self.assertEqual(mock_verify.mock_calls, []) self.assertEqual(mock_dl_list.mock_calls, []) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 05f74f9..aef780b 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -20,9 +20,9 @@ import tempfile import time import typing -import rpm import tqdm import xdg.BaseDirectory +import rpm import qubesadmin import qubesadmin.tools @@ -39,6 +39,9 @@ DATE_FMT = '%Y-%m-%d %H:%M:%S' UPDATEVM = str('global UpdateVM') +class SignatureVerificationError(Exception): + """Package signature is invalid or missing""" + def qubes_release() -> str: """Return the Qubes release.""" if os.path.exists('/usr/share/qubes/marker-vm'): @@ -540,21 +543,18 @@ def qrexec_download( raise ConnectionError( "qrexec call 'qubes.TemplateDownload' failed.") -def rpm_transactionset(key_dir: str) -> rpm.transaction.TransactionSet: - """Create RPM TransactionSet using the keys in the given directory.""" - tset = rpm.TransactionSet() - kring = rpm.keyring() +def get_keys(key_dir: str) -> typing.List[str]: + """List gpg keys""" + keys = [] for name in os.listdir(key_dir): path = os.path.join(key_dir, name) if os.path.isfile(path): - with open(path, 'rb') as fd: - kring.addKey(rpm.pubkey(fd.read())) - tset.setKeyring(kring) - return tset + keys.append(path) + return keys def verify_rpm( path: str, - transaction_set: rpm.transaction.TransactionSet, + keys: typing.List[str], nogpgcheck: bool = False ) -> rpm.hdr: """Verify the digest and signature of a RPM package and return the package @@ -566,24 +566,29 @@ def verify_rpm( case. :param path: Location of the RPM package - :param transaction_set: RPM ``TransactionSet`` :param nogpgcheck: Whether to allow invalid GPG signatures - :return: RPM package header. If verification fails, ``None`` is returned. + :return: RPM package header. If verification fails, raises an exception. """ + if not nogpgcheck: + with tempfile.TemporaryDirectory() as rpmdb_dir: + for key in keys: + subprocess.check_call( + ['rpmkeys', '--dbpath=' + rpmdb_dir, '--import', key]) + try: + output = subprocess.check_output( + ['rpmkeys', '--dbpath=' + rpmdb_dir, '--checksig', path]) + except subprocess.CalledProcessError as e: + raise SignatureVerificationError( + 'Signature verification failed: {}'.format( + e.output.decode())) + if not output.endswith(b': digests signatures OK\n'): + raise SignatureVerificationError( + 'Signature verification failed: {}'.format(output.decode())) with open(path, 'rb') as fd: - try: - hdr = transaction_set.hdrFromFdno(fd) - if hdr[rpm.RPMTAG_SIGPGP] is None \ - and hdr[rpm.RPMTAG_SIGGPG] is None: - return hdr if nogpgcheck else None - except rpm.error as e: - if str(e) == 'public key not trusted' \ - or str(e) == 'public key not available': - # FIXME: This does not work - # Should just tell TransactionSet not to verify sigs - return hdr if nogpgcheck else None - return None + tset = rpm.TransactionSet() + tset.setVSFlags(rpm.RPMVSF_MASK_NOSIGNATURES) + hdr = tset.hdrFromFdno(fd) return hdr def extract_rpm(name: str, path: str, target: str) -> bool: @@ -772,7 +777,7 @@ def install( % LOCK_FILE) try: - transaction_set = rpm_transactionset(args.keyring) + keys = get_keys(args.keyring) unverified_rpm_list = [] # rpmfile, reponame verified_rpm_list = [] @@ -784,7 +789,7 @@ def install( else: path = rpmfile - package_hdr = verify_rpm(path, transaction_set, args.nogpgcheck) + package_hdr = verify_rpm(path, keys, args.nogpgcheck) if not package_hdr: parser.error('Package \'%s\' verification failed.' % rpmfile) diff --git a/test-packages/rpm.py b/test-packages/rpm.py index c167adb..37d0730 100644 --- a/test-packages/rpm.py +++ b/test-packages/rpm.py @@ -13,6 +13,8 @@ RPMTAG_SUMMARY = 9 RPMTAG_URL = 10 RPMTAG_VERSION = 11 +RPMVSF_MASK_NOSIGNATURES = 0xc0c00 + class error(BaseException): def __init__(self, msg): self.msg = msg @@ -21,7 +23,8 @@ class error(BaseException): return self.msg class hdr(): - pass + def __getitem__(self, key): + pass class keyring(): def addKey(self, *args): @@ -31,13 +34,12 @@ class pubkey(): pass class TransactionSet(): + def setVSFlags(self, flags): + pass def setKeyring(self, *args): pass - -class transaction(): - class TransactionSet(): - def setKeyring(self, *args): - pass + def hdrFromFdno(self, fdno) -> hdr: + return hdr() def labelCompare(a, b): # Pretend that we're comparing the versions lexographically in the stub From f3954fb225fdda96d619f83e8f3321a840c4ef4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Jan 2021 22:54:24 +0100 Subject: [PATCH 083/119] qvm-template: download templates to a temporary directory Avoid risk of conflicting downloads to the same directory, reusing partial downloads, leaving broken files etc. Move template package out of temporary directory only after its verified. QubesOS/qubes-issues#2534 --- qubesadmin/tests/tools/qvm_template.py | 26 +++++++++++--------------- qubesadmin/tools/qvm_template.py | 20 +++++++++++--------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index a150c1b..365199c 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -243,14 +243,12 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock.call(args, self.app, version_selector=selector) ]) # Nothing downloaded - self.assertEqual(mock_dl.mock_calls, [ - mock.call(args, self.app, path_override='/var/cache/qvm-template', - dl_list={}, suffix='.unverified', version_selector=selector) - ]) + mock_dl.assert_called_with(args, self.app, + path_override='/var/tmp/qvm-template-tmpdir', + dl_list={}, suffix='.unverified', version_selector=selector) # Package is extracted - self.assertEqual(mock_extract.mock_calls, [ - mock.call('test-vm', path, '/var/tmp/qvm-template-tmpdir') - ]) + mock_extract.assert_called_with('test-vm', path, + '/var/tmp/qvm-template-tmpdir') # No packages overwritten, so no confirm needed self.assertEqual(mock_confirm.mock_calls, []) # qvm-template-postprocess is called @@ -363,14 +361,12 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock.call(args, self.app, version_selector=selector) ]) # Nothing downloaded - self.assertEqual(mock_dl.mock_calls, [ - mock.call(args, self.app, path_override='/var/cache/qvm-template', - dl_list={}, suffix='.unverified', version_selector=selector) - ]) + mock_dl.assert_called_with(args, self.app, + path_override='/var/tmp/qvm-template-tmpdir', + dl_list={}, suffix='.unverified', version_selector=selector) # Package is extracted - self.assertEqual(mock_extract.mock_calls, [ - mock.call('test-vm', path, '/var/tmp/qvm-template-tmpdir') - ]) + mock_extract.assert_called_with('test-vm', path, + '/var/tmp/qvm-template-tmpdir') # No packages overwritten, so no confirm needed self.assertEqual(mock_confirm.mock_calls, []) # qvm-template-postprocess is called @@ -531,7 +527,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # Nothing downloaded self.assertEqual(mock_dl.mock_calls, [ - mock.call(args, self.app, path_override='/var/cache/qvm-template', + mock.call(args, self.app, path_override='/var/tmp/qvm-template-tmpdir', dl_list={}, suffix='.unverified', version_selector=selector) ]) # Should not be executed: diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index aef780b..f16fdde 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -781,11 +781,12 @@ def install( unverified_rpm_list = [] # rpmfile, reponame verified_rpm_list = [] - def verify(rpmfile, reponame): + def verify(rpmfile, reponame, dl_dir=None): """Verify package signature and version, remove "unverified" suffix, and parse package header.""" - if reponame != '@commandline': - path = rpmfile + UNVERIFIED_SUFFIX + if dl_dir: + path = os.path.join( + dl_dir, os.path.basename(rpmfile) + UNVERIFIED_SUFFIX) else: path = rpmfile @@ -892,13 +893,14 @@ def install( 'This will override changes made in the following VMs:', override_tpls) - download(args, app, path_override=args.cachedir, - dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, - version_selector=version_selector) + with tempfile.TemporaryDirectory(dir=args.cachedir) as dl_dir: + download(args, app, path_override=dl_dir, + dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, + version_selector=version_selector) - # Verify downloaded templates - for rpmfile, reponame in unverified_rpm_list: - verify(rpmfile, reponame) + # Verify downloaded templates + for rpmfile, reponame in unverified_rpm_list: + verify(rpmfile, reponame, dl_dir=dl_dir) unverified_rpm_list = [] # Unpack and install From aeeb3daa809f6a0c1325dc31e6f09a17546e9558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Jan 2021 22:59:14 +0100 Subject: [PATCH 084/119] qvm-template: handle template extraction failure QubesOS/qubes-issues#2534 --- qubesadmin/tools/qvm_template.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index f16fdde..1ca2135 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -907,8 +907,9 @@ def install( for rpmfile, reponame, name, package_hdr in verified_rpm_list: with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: print('Installing template \'%s\'...' % name, file=sys.stderr) - # FIXME: Handle return value - extract_rpm(name, rpmfile, target) + if not extract_rpm(name, rpmfile, target): + raise Exception( + 'Failed to extract {} template'.format(name)) cmdline = [ 'qvm-template-postprocess', '--really', From 8aede943cca9c3de866687d6659b0902d284d1d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Jan 2021 23:01:47 +0100 Subject: [PATCH 085/119] qvm-template: add copyright header --- qubesadmin/tools/qvm_template.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 1ca2135..4686575 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -1,4 +1,21 @@ -#!/usr/bin/env python3 +# +# The Qubes OS Project, https://www.qubes-os.org/ +# +# Copyright (C) 2019 WillyPillow +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. '''Tool for managing VM templates.''' From 940124948a848b46873ac68799c8fcbf7bb0d54b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Jan 2021 23:02:25 +0100 Subject: [PATCH 086/119] qvm-template: minor improvements - rename parser_gen to get_parser - for consistency with other tools - clarify 'storage pool' - move '-' to the end in regex characters list QubesOS/qubes-issues#2534 --- qubesadmin/tools/qvm_template.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 4686575..9af503f 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -78,7 +78,7 @@ def qubes_release() -> str: # Return default value instead of throwing so that it works on CI return '4.1' -def parser_gen() -> argparse.ArgumentParser: +def get_parser() -> argparse.ArgumentParser: """Generate argument parser for the application.""" formatter = argparse.ArgumentDefaultsHelpFormatter parser_main = argparse.ArgumentParser(description='Qubes Template Manager', @@ -128,7 +128,7 @@ def parser_gen() -> argparse.ArgumentParser: parser_install = parser_add_command('install', help_str='Install template packages.') parser_install.add_argument('--pool', - help='Specify pool to store created VMs in.') + help='Specify storage pool to store created templates in.') parser_reinstall = parser_add_command('reinstall', help_str='Reinstall template packages.') parser_downgrade = parser_add_command('downgrade', @@ -212,7 +212,7 @@ def parser_gen() -> argparse.ArgumentParser: return parser_main -parser = parser_gen() +parser = get_parser() class TemplateState(enum.Enum): """Enum representing the state of a template.""" @@ -467,10 +467,10 @@ def qrexec_repoquery( for line in stderr.decode('ASCII').rstrip().split('\n'): print('[Qrexec] %s' % line, file=sys.stderr) raise ConnectionError("qrexec call 'qubes.TemplateSearch' failed.") - name_re = re.compile(r'^[A-Za-z0-9._+\-]*$') + name_re = re.compile(r'^[A-Za-z0-9._+-]*$') evr_re = re.compile(r'^[A-Za-z0-9._+~]*$') date_re = re.compile(r'^\d+-\d+-\d+ \d+:\d+$') - licence_re = re.compile(r'^[A-Za-z0-9._+\-()]*$') + licence_re = re.compile(r'^[A-Za-z0-9._+()-]*$') result = [] # FIXME: This breaks when \n is the first character of the description for line in stdout.split('|\n'): From b2e4d0ee34fe9a89e44dd8e1f565d41c597e304b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Jan 2021 23:03:59 +0100 Subject: [PATCH 087/119] deb,rpm: depend on qubes-repo-templates For package repos definitions + keys. --- debian/control | 1 + rpm_spec/qubes-core-admin-client.spec.in | 1 + 2 files changed, 2 insertions(+) diff --git a/debian/control b/debian/control index ff4739f..30440e3 100644 --- a/debian/control +++ b/debian/control @@ -23,6 +23,7 @@ Package: qubes-core-admin-client Architecture: any Depends: python3-qubesadmin, + qubes-repo-templates, scrypt, ${python:Depends}, ${python3:Depends}, diff --git a/rpm_spec/qubes-core-admin-client.spec.in b/rpm_spec/qubes-core-admin-client.spec.in index bcfd0a4..dc94e02 100644 --- a/rpm_spec/qubes-core-admin-client.spec.in +++ b/rpm_spec/qubes-core-admin-client.spec.in @@ -15,6 +15,7 @@ BuildRequires: python%{python3_pkgversion}-lxml BuildRequires: python%{python3_pkgversion}-xcffib Requires: python%{python3_pkgversion}-qubesadmin Requires: python%{python3_pkgversion}-yaml +Requires: qubes-repo-templates Requires: scrypt BuildArch: noarch Source0: %{name}-%{version}.tar.gz From febf014d147a67837d35984bb540997b5d951bed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Jan 2021 23:05:06 +0100 Subject: [PATCH 088/119] qvm-template-postprocess: improve data validation - validate if IP has correct syntax - print warning if value is invalid QubesOS/qubes-issues#2534 --- qubesadmin/tools/qvm_template_postprocess.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py index 36bbcdd..832fd93 100644 --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -232,6 +232,12 @@ def call_postinstall_service(vm): finally: vm.netvm = qubesadmin.DEFAULT +def validate_ip(ip): + """Check if given string has a valid IP address syntax""" + try: + return all(0 <= int(part) <= 255 for part in ip.split('.', 3)) + except ValueError: + return False @asyncio.coroutine def post_install(args): @@ -301,7 +307,11 @@ def post_install(args): 'net.fake-gateway', 'net.fake-netmask'): if key in conf: - vm.features[key] = conf[key] + if validate_ip(conf[key]): + vm.features[key] = conf[key] + else: + vm.log.warning( + 'ignoring invalid value for \'%s\'', key) if 'virt-mode' in conf: if conf['virt-mode'] == 'pv' and args.allow_pv: vm.virt_mode = 'pv' @@ -310,6 +320,8 @@ def post_install(args): '--allow-pv not set, ignoring request to change virt-mode') elif conf['virt-mode'] in ('pvh', 'hvm'): vm.virt_mode = conf['virt-mode'] + else: + vm.log.warning('ignoring invalid value for virt-mode') if 'kernel' in conf: if conf['kernel'] == '': From f1424812b0b4df7d3c89b2430e6407d36f5a9797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Jan 2021 23:16:02 +0100 Subject: [PATCH 089/119] qvm-template: improve install lock Use fcntl.flock() instead of just file existence check, so it won't fail on a stale lock file. While at it, move locking to a function decorator, to de-clutter the install function a bit. This will allow reducing indentation level, but don't do it yet, to make the patch readable. Move lock testing into a separate test, and remove it from install tests. QubesOS/qubes-issues#2534 --- qubesadmin/tests/tools/qvm_template.py | 186 ++++++------------------- qubesadmin/tools/qvm_template.py | 33 +++-- 2 files changed, 65 insertions(+), 154 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 365199c..a49f6ad 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -8,6 +8,7 @@ import pathlib import subprocess import tempfile +import fcntl import rpm import qubesadmin.tests @@ -151,13 +152,40 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) self.assertAllCalled() + @mock.patch('qubesadmin.tools.qvm_template.get_keys') + def test_090_install_lock(self, mock_get_keys): + class SuccessError(Exception): + pass + mock_get_keys.side_effect = SuccessError + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'): + with self.subTest('not locked'): + with self.assertRaises(SuccessError): + # args don't matter + qubesadmin.tools.qvm_template.install(mock.MagicMock(), None) + self.assertFalse(os.path.exists('/tmp/test.lock')) + + with self.subTest('lock exists but unlocked'): + with open('/tmp/test.lock', 'w') as f: + with self.assertRaises(SuccessError): + # args don't matter + qubesadmin.tools.qvm_template.install(mock.MagicMock(), None) + self.assertFalse(os.path.exists('/tmp/test.lock')) + with self.subTest('locked'): + with open('/tmp/test.lock', 'w') as f: + fcntl.flock(f, fcntl.LOCK_EX) + with self.assertRaises( + qubesadmin.tools.qvm_template.AlreadyRunning): + # args don't matter + qubesadmin.tools.qvm_template.install(mock.MagicMock(), None) + # and not cleaned up then + self.assertTrue(os.path.exists('/tmp/test.lock')) + def add_new_vm_side_effect(self, *args, **kwargs): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ b'0\0test-vm class=TemplateVM state=Halted\n' self.app.domains.clear_cache() return self.app.domains['test-vm'] - @mock.patch('os.remove') @mock.patch('os.rename') @mock.patch('os.makedirs') @mock.patch('subprocess.check_call') @@ -175,8 +203,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_confirm, mock_call, mock_mkdirs, - mock_rename, - mock_remove): + mock_rename): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = b'0\0' build_time = '2020-09-01 14:30:00' # 1598970600 install_time = '2020-09-01 15:30:00' @@ -213,7 +240,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_time = mock.Mock(wraps=datetime.datetime) mock_time.now.return_value = \ datetime.datetime(2020, 9, 1, 15, 30, tzinfo=datetime.timezone.utc) - with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ mock.patch('datetime.datetime', new=mock_time), \ mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ @@ -231,12 +258,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_tmpdir.return_value.__enter__.return_value = \ '/var/tmp/qvm-template-tmpdir' qubesadmin.tools.qvm_template.install(args, self.app) - # Lock file created - self.assertEqual(mock_open.mock_calls, [ - mock.call('/var/tmp/qvm-template.lck', 'x'), - mock.call().__enter__(), - mock.call().__exit__(None, None, None) - ]) # Attempt to get download list selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST self.assertEqual(mock_dl_list.mock_calls, [ @@ -269,13 +290,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # No templates downloaded, thus no renames needed self.assertEqual(mock_rename.mock_calls, []) - # Lock file removed - self.assertEqual(mock_remove.mock_calls, [ - mock.call('/var/tmp/qvm-template.lck') - ]) self.assertAllCalled() - @mock.patch('os.remove') @mock.patch('os.rename') @mock.patch('os.makedirs') @mock.patch('subprocess.check_call') @@ -293,8 +309,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_confirm, mock_call, mock_mkdirs, - mock_rename, - mock_remove): + mock_rename): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = b'0\0' build_time = '2020-09-01 14:30:00' # 1598970600 install_time = '2020-09-01 15:30:00' @@ -331,7 +346,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_time = mock.Mock(wraps=datetime.datetime) mock_time.now.return_value = \ datetime.datetime(2020, 9, 1, 15, 30, tzinfo=datetime.timezone.utc) - with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ mock.patch('datetime.datetime', new=mock_time), \ mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ @@ -349,12 +364,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_tmpdir.return_value.__enter__.return_value = \ '/var/tmp/qvm-template-tmpdir' qubesadmin.tools.qvm_template.install(args, self.app) - # Lock file created - self.assertEqual(mock_open.mock_calls, [ - mock.call('/var/tmp/qvm-template.lck', 'x'), - mock.call().__enter__(), - mock.call().__exit__(None, None, None) - ]) # Attempt to get download list selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST self.assertEqual(mock_dl_list.mock_calls, [ @@ -390,13 +399,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # No templates downloaded, thus no renames needed self.assertEqual(mock_rename.mock_calls, []) - # Lock file removed - self.assertEqual(mock_remove.mock_calls, [ - mock.call('/var/tmp/qvm-template.lck') - ]) self.assertAllCalled() - @mock.patch('os.remove') @mock.patch('os.rename') @mock.patch('os.makedirs') @mock.patch('subprocess.check_call') @@ -414,11 +418,10 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_confirm, mock_call, mock_mkdirs, - mock_rename, - mock_remove): + mock_rename): mock_verify.return_value = None mock_time = mock.Mock(wraps=datetime.datetime) - with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ mock.patch('datetime.datetime', new=mock_time), \ mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ @@ -438,12 +441,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): # Should raise parser.error with self.assertRaises(SystemExit): qubesadmin.tools.qvm_template.install(args, self.app) - # Lock file created - self.assertEqual(mock_open.mock_calls, [ - mock.call('/var/tmp/qvm-template.lck', 'x'), - mock.call().__enter__(), - mock.call().__exit__(None, None, None) - ]) # Check error message self.assertTrue('verification failed' in mock_err.getvalue()) # Should not be executed: @@ -453,13 +450,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): self.assertEqual(mock_confirm.mock_calls, []) self.assertEqual(mock_call.mock_calls, []) self.assertEqual(mock_rename.mock_calls, []) - # Lock file removed - self.assertEqual(mock_remove.mock_calls, [ - mock.call('/var/tmp/qvm-template.lck') - ]) self.assertAllCalled() - @mock.patch('os.remove') @mock.patch('os.rename') @mock.patch('os.makedirs') @mock.patch('subprocess.check_call') @@ -477,8 +469,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_confirm, mock_call, mock_mkdirs, - mock_rename, - mock_remove): + mock_rename): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ b'0\0test-vm class=TemplateVM state=Halted\n' mock_verify.return_value = { @@ -494,7 +485,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): } mock_dl_list.return_value = {} mock_time = mock.Mock(wraps=datetime.datetime) - with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ mock.patch('datetime.datetime', new=mock_time), \ mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ @@ -512,12 +503,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_tmpdir.return_value.__enter__.return_value = \ '/var/tmp/qvm-template-tmpdir' qubesadmin.tools.qvm_template.install(args, self.app) - # Lock file created - self.assertEqual(mock_open.mock_calls, [ - mock.call('/var/tmp/qvm-template.lck', 'x'), - mock.call().__enter__(), - mock.call().__exit__(None, None, None) - ]) # Check warning message self.assertTrue('already installed' in mock_err.getvalue()) # Attempt to get download list @@ -535,13 +520,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): self.assertEqual(mock_confirm.mock_calls, []) self.assertEqual(mock_call.mock_calls, []) self.assertEqual(mock_rename.mock_calls, []) - # Lock file removed - self.assertEqual(mock_remove.mock_calls, [ - mock.call('/var/tmp/qvm-template.lck') - ]) self.assertAllCalled() - @mock.patch('os.remove') @mock.patch('os.rename') @mock.patch('os.makedirs') @mock.patch('subprocess.check_call') @@ -559,8 +539,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_confirm, mock_call, mock_mkdirs, - mock_rename, - mock_remove): + mock_rename): mock_verify.return_value = { rpm.RPMTAG_NAME : 'Xqubes-template-test-vm', rpm.RPMTAG_BUILDTIME : 1598970600, @@ -573,7 +552,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): rpm.RPMTAG_VERSION : '4.1' } mock_time = mock.Mock(wraps=datetime.datetime) - with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ mock.patch('datetime.datetime', new=mock_time), \ mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ @@ -592,12 +571,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): '/var/tmp/qvm-template-tmpdir' with self.assertRaises(SystemExit): qubesadmin.tools.qvm_template.install(args, self.app) - # Lock file created - self.assertEqual(mock_open.mock_calls, [ - mock.call('/var/tmp/qvm-template.lck', 'x'), - mock.call().__enter__(), - mock.call().__exit__(None, None, None) - ]) # Check error message self.assertTrue('Illegal package name' in mock_err.getvalue()) # Should not be executed: @@ -607,74 +580,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): self.assertEqual(mock_confirm.mock_calls, []) self.assertEqual(mock_call.mock_calls, []) self.assertEqual(mock_rename.mock_calls, []) - # Lock file removed - self.assertEqual(mock_remove.mock_calls, [ - mock.call('/var/tmp/qvm-template.lck') - ]) self.assertAllCalled() - @mock.patch('os.rename') - @mock.patch('os.makedirs') - @mock.patch('subprocess.check_call') - @mock.patch('qubesadmin.tools.qvm_template.confirm_action') - @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') - @mock.patch('qubesadmin.tools.qvm_template.download') - @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') - @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') - @mock.patch('qubesadmin.tools.qvm_template.rpm_transactionset') - def test_105_install_local_existinginstance_fail( - self, - mock_ts, - mock_verify, - mock_dl_list, - mock_dl, - mock_extract, - mock_confirm, - mock_call, - mock_mkdirs, - mock_rename): - mock_time = mock.Mock(wraps=datetime.datetime) - with mock.patch('datetime.datetime', new=mock_time), \ - mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ - mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ - tempfile.NamedTemporaryFile(suffix='.rpm') as template_file: - path = template_file.name - args = argparse.Namespace( - templates=[path], - keyring='/usr/share/qubes/repo-templates/keys', - nogpgcheck=False, - cachedir='/var/cache/qvm-template', - yes=False, - allow_pv=False, - pool=None - ) - mock_tmpdir.return_value.__enter__.return_value = \ - '/var/tmp/qvm-template-tmpdir' - pathlib.Path('/var/tmp/qvm-template.lck').touch() - try: - with mock.patch('os.remove') as mock_remove: - with self.assertRaises(SystemExit): - qubesadmin.tools.qvm_template.install(args, self.app) - self.assertEqual(mock_remove.mock_calls, []) - finally: - # Lock file not removed - self.assertTrue(os.path.exists('/var/tmp/qvm-template.lck')) - os.remove('/var/tmp/qvm-template.lck') - # Check error message - self.assertTrue('another instance of qvm-template is running' \ - in mock_err.getvalue()) - # Should not be executed: - self.assertEqual(mock_ts.mock_calls, []) - self.assertEqual(mock_verify.mock_calls, []) - self.assertEqual(mock_dl_list.mock_calls, []) - self.assertEqual(mock_dl.mock_calls, []) - self.assertEqual(mock_extract.mock_calls, []) - self.assertEqual(mock_confirm.mock_calls, []) - self.assertEqual(mock_call.mock_calls, []) - self.assertEqual(mock_rename.mock_calls, []) - self.assertAllCalled() - - @mock.patch('os.remove') @mock.patch('os.rename') @mock.patch('os.makedirs') @mock.patch('subprocess.check_call') @@ -692,10 +599,9 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_confirm, mock_call, mock_mkdirs, - mock_rename, - mock_remove): + mock_rename): mock_time = mock.Mock(wraps=datetime.datetime) - with mock.patch('builtins.open', mock.mock_open()) as mock_open, \ + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ mock.patch('datetime.datetime', new=mock_time), \ mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ mock.patch('sys.stderr', new=io.StringIO()) as mock_err: @@ -713,12 +619,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): '/var/tmp/qvm-template-tmpdir' with self.assertRaises(SystemExit): qubesadmin.tools.qvm_template.install(args, self.app) - # Lock file created - self.assertEqual(mock_open.mock_calls, [ - mock.call('/var/tmp/qvm-template.lck', 'x'), - mock.call().__enter__(), - mock.call().__exit__(None, None, None) - ]) # Check error message self.assertTrue(f"RPM file '{path}' not found" \ in mock_err.getvalue()) @@ -730,10 +630,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): self.assertEqual(mock_confirm.mock_calls, []) self.assertEqual(mock_call.mock_calls, []) self.assertEqual(mock_rename.mock_calls, []) - # Lock file removed - self.assertEqual(mock_remove.mock_calls, [ - mock.call('/var/tmp/qvm-template.lck') - ]) self.assertAllCalled() def test_110_qrexec_payload_refresh_success(self): diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 9af503f..6877559 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -23,6 +23,7 @@ import argparse import collections import datetime import enum +import fcntl import fnmatch import functools import itertools @@ -56,6 +57,9 @@ DATE_FMT = '%Y-%m-%d %H:%M:%S' UPDATEVM = str('global UpdateVM') +class AlreadyRunning(Exception): + """Another qvm-template is already running""" + class SignatureVerificationError(Exception): """Package signature is invalid or missing""" @@ -768,6 +772,25 @@ def download( print('Error: \'%s\' download failed.' % spec, file=sys.stderr) sys.exit(1) +def locked(func): + """Execute given function under a lock in *LOCK_FILE*""" + @functools.wraps(func) + def wrapper(*args, **kwargs): + with open(LOCK_FILE, 'w') as lock: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + raise AlreadyRunning( + ('cannot get lock on %s. ' + 'Perhaps another instance of qvm-template is running?') + % LOCK_FILE) + try: + return func(*args, **kwargs) + finally: + os.remove(LOCK_FILE) + return wrapper + +@locked def install( args: argparse.Namespace, app: qubesadmin.app.QubesBase, @@ -785,14 +808,6 @@ def install( :param override_existing: Whether to override existing packages. Used for reinstall, upgrade, and downgrade operations """ - try: - with open(LOCK_FILE, 'x') as _: - pass - except FileExistsError: - parser.error(('%s already exists.' - ' Perhaps another instance of qvm-template is running?') - % LOCK_FILE) - try: keys = get_keys(args.keyring) @@ -969,7 +984,7 @@ def install( tpl.features['template-description'] = \ package_hdr[rpm.RPMTAG_DESCRIPTION].replace('\n', '|') finally: - os.remove(LOCK_FILE) + pass def list_templates(args: argparse.Namespace, app: qubesadmin.app.QubesBase, operation: str) -> None: From fe369ce523bffa82b245ee8c9529a95c94e5cab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 30 Jan 2021 05:39:59 +0100 Subject: [PATCH 090/119] qvm-template: cleanup install function Remove now unused try/finally in install() and reduce indentation. No functional change. --- qubesadmin/tools/qvm_template.py | 325 +++++++++++++++---------------- 1 file changed, 161 insertions(+), 164 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 6877559..cb65b4d 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -808,183 +808,180 @@ def install( :param override_existing: Whether to override existing packages. Used for reinstall, upgrade, and downgrade operations """ - try: - keys = get_keys(args.keyring) + keys = get_keys(args.keyring) - unverified_rpm_list = [] # rpmfile, reponame - verified_rpm_list = [] - def verify(rpmfile, reponame, dl_dir=None): - """Verify package signature and version, remove "unverified" - suffix, and parse package header.""" - if dl_dir: - path = os.path.join( - dl_dir, os.path.basename(rpmfile) + UNVERIFIED_SUFFIX) - else: - path = rpmfile + unverified_rpm_list = [] # rpmfile, reponame + verified_rpm_list = [] + def verify(rpmfile, reponame, dl_dir=None): + """Verify package signature and version, remove "unverified" + suffix, and parse package header.""" + if dl_dir: + path = os.path.join( + dl_dir, os.path.basename(rpmfile) + UNVERIFIED_SUFFIX) + else: + path = rpmfile - package_hdr = verify_rpm(path, keys, args.nogpgcheck) - if not package_hdr: - parser.error('Package \'%s\' verification failed.' % rpmfile) + package_hdr = verify_rpm(path, keys, args.nogpgcheck) + if not package_hdr: + parser.error('Package \'%s\' verification failed.' % rpmfile) - package_name = package_hdr[rpm.RPMTAG_NAME] - if not package_name.startswith(PACKAGE_NAME_PREFIX): + package_name = package_hdr[rpm.RPMTAG_NAME] + if not package_name.startswith(PACKAGE_NAME_PREFIX): + parser.error( + 'Illegal package name for package \'%s\'.' % rpmfile) + # Remove prefix to get the real template name + name = package_name[len(PACKAGE_NAME_PREFIX):] + + if path != rpmfile: + os.rename(path, rpmfile) + + # Check if already installed + if not override_existing and name in app.domains: + print(('Template \'%s\' already installed, skipping...' + ' (You may want to use the' + ' {reinstall,upgrade,downgrade}' + ' operations.)') % name, file=sys.stderr) + return + + # Check if version is really what we want + if override_existing: + vm = get_managed_template_vm(app, name) + pkg_evr = ( + str(package_hdr[rpm.RPMTAG_EPOCHNUM]), + package_hdr[rpm.RPMTAG_VERSION], + package_hdr[rpm.RPMTAG_RELEASE]) + vm_evr = query_local_evr(vm) + cmp_res = rpm.labelCompare(pkg_evr, vm_evr) + if version_selector == VersionSelector.REINSTALL \ + and cmp_res != 0: parser.error( - 'Illegal package name for package \'%s\'.' % rpmfile) - # Remove prefix to get the real template name - name = package_name[len(PACKAGE_NAME_PREFIX):] - - if path != rpmfile: - os.rename(path, rpmfile) - - # Check if already installed - if not override_existing and name in app.domains: - print(('Template \'%s\' already installed, skipping...' - ' (You may want to use the' - ' {reinstall,upgrade,downgrade}' - ' operations.)') % name, file=sys.stderr) + 'Same version of template \'%s\' not found.' \ + % name) + elif version_selector == VersionSelector.LATEST_LOWER \ + and cmp_res != -1: + print(("Template '%s' of lower version" + " already installed, skipping..." % name), + file=sys.stderr) + return + elif version_selector == VersionSelector.LATEST_HIGHER \ + and cmp_res != 1: + print(("Template '%s' of higher version" + " already installed, skipping..." % name), + file=sys.stderr) return - # Check if version is really what we want - if override_existing: - vm = get_managed_template_vm(app, name) - pkg_evr = ( - str(package_hdr[rpm.RPMTAG_EPOCHNUM]), - package_hdr[rpm.RPMTAG_VERSION], - package_hdr[rpm.RPMTAG_RELEASE]) - vm_evr = query_local_evr(vm) - cmp_res = rpm.labelCompare(pkg_evr, vm_evr) - if version_selector == VersionSelector.REINSTALL \ - and cmp_res != 0: - parser.error( - 'Same version of template \'%s\' not found.' \ - % name) - elif version_selector == VersionSelector.LATEST_LOWER \ - and cmp_res != -1: - print(("Template '%s' of lower version" - " already installed, skipping..." % name), - file=sys.stderr) - return - elif version_selector == VersionSelector.LATEST_HIGHER \ - and cmp_res != 1: - print(("Template '%s' of higher version" - " already installed, skipping..." % name), - file=sys.stderr) - return + verified_rpm_list.append((rpmfile, reponame, name, package_hdr)) - verified_rpm_list.append((rpmfile, reponame, name, package_hdr)) + # Process local templates + for template in args.templates: + if template.endswith('.rpm'): + if not os.path.exists(template): + parser.error('RPM file \'%s\' not found.' % template) + unverified_rpm_list.append((template, '@commandline')) - # Process local templates - for template in args.templates: - if template.endswith('.rpm'): - if not os.path.exists(template): - parser.error('RPM file \'%s\' not found.' % template) - unverified_rpm_list.append((template, '@commandline')) + # First verify local RPMs and extract header + for rpmfile, reponame in unverified_rpm_list: + verify(rpmfile, reponame) + unverified_rpm_list = [] - # First verify local RPMs and extract header + os.makedirs(args.cachedir, exist_ok=True) + + # Get list of templates to download + dl_list = get_dl_list(args, app, version_selector=version_selector) + dl_list_copy = dl_list.copy() + for name, entry in dl_list.items(): + # Should be ensured by checks in repoquery + assert entry.reponame != '@commandline' + # Verify that the templates to be downloaded are not yet installed + # Note that we *still* have to do this again in verify() for + # already-downloaded templates + if not override_existing and name in app.domains: + print(('Template \'%s\' already installed, skipping...' + ' (You may want to use the' + ' {reinstall,upgrade,downgrade}' + ' operations.)') % name, file=sys.stderr) + del dl_list_copy[name] + else: + # XXX: Perhaps this is better returned by download() + version_str = build_version_str(entry.evr) + target_file = \ + '%s%s-%s.rpm' % (PACKAGE_NAME_PREFIX, name, version_str) + unverified_rpm_list.append( + (os.path.join(args.cachedir, target_file), entry.reponame)) + dl_list = dl_list_copy + + # Ask the user for confirmation before we actually download stuff + if override_existing and not args.yes: + override_tpls = [] + # Local templates, already verified + for _, _, name, _ in verified_rpm_list: + override_tpls.append(name) + # Templates not yet downloaded + for name in dl_list: + override_tpls.append(name) + + confirm_action( + 'This will override changes made in the following VMs:', + override_tpls) + + with tempfile.TemporaryDirectory(dir=args.cachedir) as dl_dir: + download(args, app, path_override=dl_dir, + dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, + version_selector=version_selector) + + # Verify downloaded templates for rpmfile, reponame in unverified_rpm_list: - verify(rpmfile, reponame) - unverified_rpm_list = [] + verify(rpmfile, reponame, dl_dir=dl_dir) + unverified_rpm_list = [] - os.makedirs(args.cachedir, exist_ok=True) + # Unpack and install + for rpmfile, reponame, name, package_hdr in verified_rpm_list: + with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: + print('Installing template \'%s\'...' % name, file=sys.stderr) + if not extract_rpm(name, rpmfile, target): + raise Exception( + 'Failed to extract {} template'.format(name)) + cmdline = [ + 'qvm-template-postprocess', + '--really', + '--no-installed-by-rpm', + ] + if args.allow_pv: + cmdline.append('--allow-pv') + if not override_existing and args.pool: + cmdline += ['--pool', args.pool] + subprocess.check_call(cmdline + [ + 'post-install', + name, + target + PATH_PREFIX + '/' + name]) - # Get list of templates to download - dl_list = get_dl_list(args, app, version_selector=version_selector) - dl_list_copy = dl_list.copy() - for name, entry in dl_list.items(): - # Should be ensured by checks in repoquery - assert entry.reponame != '@commandline' - # Verify that the templates to be downloaded are not yet installed - # Note that we *still* have to do this again in verify() for - # already-downloaded templates - if not override_existing and name in app.domains: - print(('Template \'%s\' already installed, skipping...' - ' (You may want to use the' - ' {reinstall,upgrade,downgrade}' - ' operations.)') % name, file=sys.stderr) - del dl_list_copy[name] - else: - # XXX: Perhaps this is better returned by download() - version_str = build_version_str(entry.evr) - target_file = \ - '%s%s-%s.rpm' % (PACKAGE_NAME_PREFIX, name, version_str) - unverified_rpm_list.append( - (os.path.join(args.cachedir, target_file), entry.reponame)) - dl_list = dl_list_copy + app.domains.refresh_cache(force=True) + tpl = app.domains[name] - # Ask the user for confirmation before we actually download stuff - if override_existing and not args.yes: - override_tpls = [] - # Local templates, already verified - for _, _, name, _ in verified_rpm_list: - override_tpls.append(name) - # Templates not yet downloaded - for name in dl_list: - override_tpls.append(name) - - confirm_action( - 'This will override changes made in the following VMs:', - override_tpls) - - with tempfile.TemporaryDirectory(dir=args.cachedir) as dl_dir: - download(args, app, path_override=dl_dir, - dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, - version_selector=version_selector) - - # Verify downloaded templates - for rpmfile, reponame in unverified_rpm_list: - verify(rpmfile, reponame, dl_dir=dl_dir) - unverified_rpm_list = [] - - # Unpack and install - for rpmfile, reponame, name, package_hdr in verified_rpm_list: - with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: - print('Installing template \'%s\'...' % name, file=sys.stderr) - if not extract_rpm(name, rpmfile, target): - raise Exception( - 'Failed to extract {} template'.format(name)) - cmdline = [ - 'qvm-template-postprocess', - '--really', - '--no-installed-by-rpm', - ] - if args.allow_pv: - cmdline.append('--allow-pv') - if not override_existing and args.pool: - cmdline += ['--pool', args.pool] - subprocess.check_call(cmdline + [ - 'post-install', - name, - target + PATH_PREFIX + '/' + name]) - - app.domains.refresh_cache(force=True) - tpl = app.domains[name] - - tpl.features['template-name'] = name - tpl.features['template-epoch'] = \ - package_hdr[rpm.RPMTAG_EPOCHNUM] - tpl.features['template-version'] = \ - package_hdr[rpm.RPMTAG_VERSION] - tpl.features['template-release'] = \ - package_hdr[rpm.RPMTAG_RELEASE] - tpl.features['template-reponame'] = reponame - tpl.features['template-buildtime'] = \ - datetime.datetime.fromtimestamp( - int(package_hdr[rpm.RPMTAG_BUILDTIME]), - tz=datetime.timezone.utc) \ - .strftime(DATE_FMT) - tpl.features['template-installtime'] = \ - datetime.datetime.now( - tz=datetime.timezone.utc).strftime(DATE_FMT) - tpl.features['template-license'] = \ - package_hdr[rpm.RPMTAG_LICENSE] - tpl.features['template-url'] = \ - package_hdr[rpm.RPMTAG_URL] - tpl.features['template-summary'] = \ - package_hdr[rpm.RPMTAG_SUMMARY] - tpl.features['template-description'] = \ - package_hdr[rpm.RPMTAG_DESCRIPTION].replace('\n', '|') - finally: - pass + tpl.features['template-name'] = name + tpl.features['template-epoch'] = \ + package_hdr[rpm.RPMTAG_EPOCHNUM] + tpl.features['template-version'] = \ + package_hdr[rpm.RPMTAG_VERSION] + tpl.features['template-release'] = \ + package_hdr[rpm.RPMTAG_RELEASE] + tpl.features['template-reponame'] = reponame + tpl.features['template-buildtime'] = \ + datetime.datetime.fromtimestamp( + int(package_hdr[rpm.RPMTAG_BUILDTIME]), + tz=datetime.timezone.utc) \ + .strftime(DATE_FMT) + tpl.features['template-installtime'] = \ + datetime.datetime.now( + tz=datetime.timezone.utc).strftime(DATE_FMT) + tpl.features['template-license'] = \ + package_hdr[rpm.RPMTAG_LICENSE] + tpl.features['template-url'] = \ + package_hdr[rpm.RPMTAG_URL] + tpl.features['template-summary'] = \ + package_hdr[rpm.RPMTAG_SUMMARY] + tpl.features['template-description'] = \ + package_hdr[rpm.RPMTAG_DESCRIPTION].replace('\n', '|') def list_templates(args: argparse.Namespace, app: qubesadmin.app.QubesBase, operation: str) -> None: From f053f51644c06f200009c365281fdb07b852e5f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 30 Jan 2021 06:07:24 +0100 Subject: [PATCH 091/119] qvm-template: remove downloaded package after installation At least by default. Otherwise they will pile up in the cache dir. --- qubesadmin/tools/qvm_template.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index cb65b4d..c99ddac 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -124,6 +124,8 @@ def get_parser() -> argparse.ArgumentParser: help='Set repository metadata as expired before running the command.') parser_main.add_argument('--cachedir', default=CACHE_DIR, help='Specify cache directory.') + parser_main.add_argument('--keep-cache', default=False, + help='Keep downloaded packages in cache dir') parser_main.add_argument('--yes', action='store_true', help='Assume "yes" to questions.') parser_main.add_argument('--quiet', action='store_true', @@ -982,6 +984,8 @@ def install( package_hdr[rpm.RPMTAG_SUMMARY] tpl.features['template-description'] = \ package_hdr[rpm.RPMTAG_DESCRIPTION].replace('\n', '|') + if rpmfile.startswith(args.cachedir) and not args.keep_cache: + os.remove(rpmfile) def list_templates(args: argparse.Namespace, app: qubesadmin.app.QubesBase, operation: str) -> None: From ed6aff3b1e1ada20c2a22c2e62033f3b1e09061e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 30 Jan 2021 06:07:56 +0100 Subject: [PATCH 092/119] qvm-template-preprocess: remove confusing message Don't confuse user during normal template install. --- qubesadmin/tools/qvm_template_postprocess.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py index 832fd93..b5e04c3 100644 --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -374,7 +374,6 @@ def is_chroot(): stat_root.st_dev != stat_init_root.st_dev or stat_root.st_ino != stat_init_root.st_ino) except IOError: - print('Stat failed, assuming not chroot', file=sys.stderr) return False From 6980e7ba14981496c66fb520bf330f7f6f348721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 3 Feb 2021 02:15:26 +0100 Subject: [PATCH 093/119] Store template repo configuration in /etc This way it's easier to permanently enable repos. --- qubesadmin/tests/tools/qvm_template.py | 18 +++++++++--------- qubesadmin/tools/qvm_template.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index a49f6ad..348b39c 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -645,7 +645,7 @@ enabled = 1 fastestmirror = 1 metadata_expire = 7d gpgcheck = 1 -gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary ''' repo_str2 = \ '''[qubes-templates-itl-testing] @@ -656,7 +656,7 @@ metalink = https://yum.qubes-os.org/r$releasever/templates-itl-testing/repodata/ enabled = 0 fastestmirror = 1 gpgcheck = 1 -gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary ''' repo_conf1.write(repo_str1.encode()) repo_conf1.flush() @@ -696,7 +696,7 @@ enabled = 1 fastestmirror = 1 metadata_expire = 7d gpgcheck = 1 -gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary ''' repo_conf1.write(repo_str1.encode()) repo_conf1.flush() @@ -730,7 +730,7 @@ enabled = 1 fastestmirror = 1 metadata_expire = 7d gpgcheck = 1 -gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary ''' repo_conf1.write(repo_str1.encode()) repo_conf1.flush() @@ -764,7 +764,7 @@ enabled = 1 fastestmirror = 1 metadata_expire = 7d gpgcheck = 1 -gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary ''' repo_conf1.write(repo_str1.encode()) repo_conf1.flush() @@ -798,7 +798,7 @@ enabled = 1 fastestmirror = 1 metadata_expire = 7d gpgcheck = 1 -gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary ''' repo_conf1.write(repo_str1.encode()) repo_conf1.flush() @@ -832,7 +832,7 @@ enabled = 1 fastestmirror = 1 metadata_expire = 7d gpgcheck = 1 -gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary ''' repo_conf1.write(repo_str1.encode()) repo_conf1.flush() @@ -866,7 +866,7 @@ enabled = 1 fastestmirror = 1 metadata_expire = 7d gpgcheck = 1 -gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary ''' repo_conf1.write(repo_str1.encode()) repo_conf1.flush() @@ -900,7 +900,7 @@ enabled = 1 fastestmirror = 1 metadata_expire = 7d gpgcheck = 1 -gpgkey = file:///usr/share/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary ''' repo_conf1.write(repo_str1.encode()) repo_conf1.flush() diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index c99ddac..4fac98b 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -98,11 +98,11 @@ def get_parser() -> argparse.ArgumentParser: description=help_str) parser_main.add_argument('--repo-files', action='append', - default=['/usr/share/qubes/repo-templates/qubes-templates.repo'], + default=['/etc/qubes/repo-templates/qubes-templates.repo'], help=('Specify files containing DNF repository configuration.' ' Can be used more than once.')) parser_main.add_argument('--keyring', - default='/usr/share/qubes/repo-templates/keys', + default='/etc/qubes/repo-templates/keys', help='Specify directory containing RPM public keys.') parser_main.add_argument('--updatevm', default=UPDATEVM, help=('Specify VM to download updates from.' From e0063d880844f555309703147721705cccfc3e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 4 Feb 2021 14:03:10 +0100 Subject: [PATCH 094/119] qvm-template: use QubesArgumentParser It produces consistent help for subcommands and already handles --verbose/--quiet. --- qubesadmin/tools/qvm_template.py | 64 ++++++++++++++++---------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 4fac98b..f63e267 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -85,9 +85,11 @@ def qubes_release() -> str: def get_parser() -> argparse.ArgumentParser: """Generate argument parser for the application.""" formatter = argparse.ArgumentDefaultsHelpFormatter - parser_main = argparse.ArgumentParser(description='Qubes Template Manager', + parser_main = qubesadmin.tools.QubesArgumentParser(description=__doc__, formatter_class=formatter) - subparsers = parser_main.add_subparsers(dest='operation', + parser_main.register('action', 'parsers', + qubesadmin.tools.AliasedSubParsersAction) + subparsers = parser_main.add_subparsers(dest='command', description='Command to run.') def parser_add_command(cmd, help_str): @@ -124,12 +126,10 @@ def get_parser() -> argparse.ArgumentParser: help='Set repository metadata as expired before running the command.') parser_main.add_argument('--cachedir', default=CACHE_DIR, help='Specify cache directory.') - parser_main.add_argument('--keep-cache', default=False, + parser_main.add_argument('--keep-cache', action='store_true', default=False, help='Keep downloaded packages in cache dir') parser_main.add_argument('--yes', action='store_true', help='Assume "yes" to questions.') - parser_main.add_argument('--quiet', action='store_true', - help='Decrease verbosity.') # qvm-template {install,reinstall,downgrade,upgrade} parser_install = parser_add_command('install', help_str='Install template packages.') @@ -359,7 +359,7 @@ def confirm_action(msg: str, affected: typing.List[str]) -> None: while confirm != 'y': confirm = input('Are you sure? [y/N] ').lower() if confirm == 'n': - print('Operation cancelled.') + print('command cancelled.') sys.exit(1) def qrexec_popen( @@ -988,12 +988,12 @@ def install( os.remove(rpmfile) def list_templates(args: argparse.Namespace, - app: qubesadmin.app.QubesBase, operation: str) -> None: + app: qubesadmin.app.QubesBase, command: str) -> None: """Command that lists templates. :param args: Arguments received by the application. :param app: Qubes application object - :param operation: If set to ``list``, display a listing similar to ``dnf + :param command: If set to ``list``, display a listing similar to ``dnf list``. If set to ``info``, display detailed template information similar to ``dnf info``. Otherwise, an ``AssertionError`` is raised. """ @@ -1080,12 +1080,12 @@ def list_templates(args: argparse.Namespace, outputs[status.value] = output return outputs - if operation == 'list': + if command == 'list': append = append_list - elif operation == 'info': + elif command == 'info': append = append_info else: - assert False and 'Unknown operation' + assert False and 'Unknown command' def append_vm(vm, status): append(query_local(vm), status, vm.features['template-installtime']) @@ -1143,24 +1143,24 @@ def list_templates(args: argparse.Namespace, parser.error('No matching templates to list') if args.machine_readable: - if operation == 'info': + if command == 'info': tpl_list_dict = info_to_machine_output(tpl_list) - elif operation == 'list': + elif command == 'list': tpl_list_dict = list_to_machine_output(tpl_list) for status, grp in tpl_list_dict.items(): for line in grp: print('|'.join([status] + list(line.values()))) elif args.machine_readable_json: - if operation == 'info': + if command == 'info': tpl_list_dict = \ info_to_machine_output(tpl_list, replace_newline=False) - elif operation == 'list': + elif command == 'list': tpl_list_dict = list_to_machine_output(tpl_list) print(json.dumps(tpl_list_dict)) else: - if operation == 'info': + if command == 'info': tpl_list = info_to_human_output(tpl_list) - elif operation == 'list': + elif command == 'list': tpl_list = list_to_human_output(tpl_list) for status, grp in tpl_list: print(status.title()) @@ -1416,8 +1416,8 @@ def main(args: typing.Optional[typing.Sequence[str]] = None, """ p_args = parser.parse_args(args) - if not p_args.operation: - parser.error('An operation needs to be specified.') + if not p_args.command: + parser.error('A command needs to be specified.') # If the user specified other repo files... if len(p_args.repo_files) > 1: @@ -1433,35 +1433,35 @@ def main(args: typing.Optional[typing.Sequence[str]] = None, if p_args.refresh: qrexec_repoquery(p_args, app, refresh=True) - if p_args.operation == 'download': + if p_args.command == 'download': download(p_args, app) - elif p_args.operation == 'install': + elif p_args.command == 'install': install(p_args, app) - elif p_args.operation == 'reinstall': + elif p_args.command == 'reinstall': install(p_args, app, version_selector=VersionSelector.REINSTALL, override_existing=True) - elif p_args.operation == 'downgrade': + elif p_args.command == 'downgrade': install(p_args, app, version_selector=VersionSelector.LATEST_LOWER, override_existing=True) - elif p_args.operation == 'upgrade': + elif p_args.command == 'upgrade': install(p_args, app, version_selector=VersionSelector.LATEST_HIGHER, override_existing=True) - elif p_args.operation == 'list': + elif p_args.command == 'list': list_templates(p_args, app, 'list') - elif p_args.operation == 'info': + elif p_args.command == 'info': list_templates(p_args, app, 'info') - elif p_args.operation == 'search': + elif p_args.command == 'search': search(p_args, app) - elif p_args.operation == 'remove': + elif p_args.command == 'remove': remove(p_args, app, disassoc=p_args.disassoc) - elif p_args.operation == 'purge': + elif p_args.command == 'purge': remove(p_args, app, purge=True) - elif p_args.operation == 'clean': + elif p_args.command == 'clean': clean(p_args, app) - elif p_args.operation == 'repolist': + elif p_args.command == 'repolist': repolist(p_args, app) else: - parser.error('Operation \'%s\' not supported.' % p_args.operation) + parser.error('Command \'%s\' not supported.' % p_args.command) return 0 From 10bea1b77e15429120f726aa1ca384d1439254ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 4 Feb 2021 17:04:48 +0100 Subject: [PATCH 095/119] qvm-template: allow global arguments after action name It's convenient to use for example `qvm-template list --enablerepo=*-testing` Previously, _some_ options needed to be before action name. --- qubesadmin/tools/qvm_template.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index f63e267..731ea16 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -1414,7 +1414,9 @@ def main(args: typing.Optional[typing.Sequence[str]] = None, :return: Return code of the application """ - p_args = parser.parse_args(args) + # do two passes to allow global options after command name too + p_args, args = parser.parse_known_args(args) + p_args = parser.parse_args(args, p_args) if not p_args.command: parser.error('A command needs to be specified.') From 86326b53c4a5415d26a9bf95b8dfe8a2f1d9bad8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 6 Feb 2021 03:34:39 +0100 Subject: [PATCH 096/119] qvm-template: factor filter_version() out of get_dl_list() This allows reusing version filtering (getting only a single version per template) in other places. For equal versions packages, prefer the one from non-testing repository. --- qubesadmin/tools/qvm_template.py | 92 ++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 731ea16..0daeab9 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -263,6 +263,11 @@ class Template(typing.NamedTuple): summary: str description: str + @property + def evr(self): + """Return a tuple of (EPOCH, VERSION, RELEASE)""" + return self.epoch, self.version, self.release + class DlEntry(typing.NamedTuple): """Information about a template to be downloaded.""" evr: typing.Tuple[str, str, str] @@ -634,6 +639,50 @@ def extract_rpm(name: str, path: str, target: str) -> bool: ], stdin=rpm2cpio.stdout, stdout=subprocess.DEVNULL) return rpm2cpio.wait() == 0 and cpio.wait() == 0 + +def filter_version( + query_res, + app: qubesadmin.app.QubesBase, + version_selector: VersionSelector = VersionSelector.LATEST): + """Select only one version for given template name""" + # We only select one package for each distinct package name + results: typing.Dict[str, Template] = {} + + for entry in query_res: + evr = (entry.epoch, entry.version, entry.release) + insert = False + if version_selector == VersionSelector.LATEST: + if entry.name not in results: + insert = True + if entry.name in results \ + and rpm.labelCompare(results[entry.name].evr, evr) < 0: + insert = True + if entry.name in results \ + and rpm.labelCompare(results[entry.name].evr, evr) == 0 \ + and 'testing' not in entry.reponame: + # for the same-version matches, prefer non-testing one + insert = True + elif version_selector == VersionSelector.REINSTALL: + vm = get_managed_template_vm(app, entry.name) + cur_ver = query_local_evr(vm) + if rpm.labelCompare(evr, cur_ver) == 0: + insert = True + elif version_selector in [VersionSelector.LATEST_LOWER, + VersionSelector.LATEST_HIGHER]: + vm = get_managed_template_vm(app, entry.name) + cur_ver = query_local_evr(vm) + cmp_res = -1 \ + if version_selector == VersionSelector.LATEST_LOWER \ + else 1 + if rpm.labelCompare(evr, cur_ver) == cmp_res: + if entry.name not in results \ + or rpm.labelCompare(results[entry.name].evr, evr) < 0: + insert = True + if insert: + results[entry.name] = entry + + return results.values() + def get_dl_list( args: argparse.Namespace, app: qubesadmin.app.QubesBase, @@ -651,10 +700,6 @@ def get_dl_list( """ full_candid: typing.Dict[str, DlEntry] = {} for template in args.templates: - # This will be merged into `full_candid` later. - # It is separated so that we can check whether it is empty. - candid: typing.Dict[str, DlEntry] = {} - # Skip local RPMs if template.endswith('.rpm'): continue @@ -662,35 +707,10 @@ def get_dl_list( query_res = qrexec_repoquery(args, app, PACKAGE_NAME_PREFIX + template) # We only select one package for each distinct package name - for entry in query_res: - ver = (entry.epoch, entry.version, entry.release) - insert = False - if version_selector == VersionSelector.LATEST: - if entry.name not in candid \ - or rpm.labelCompare(candid[entry.name][0], ver) < 0: - insert = True - elif version_selector == VersionSelector.REINSTALL: - vm = get_managed_template_vm(app, entry.name) - cur_ver = query_local_evr(vm) - if rpm.labelCompare(ver, cur_ver) == 0: - insert = True - elif version_selector in [VersionSelector.LATEST_LOWER, - VersionSelector.LATEST_HIGHER]: - vm = get_managed_template_vm(app, entry.name) - cur_ver = query_local_evr(vm) - cmp_res = -1 \ - if version_selector == VersionSelector.LATEST_LOWER \ - else 1 - if rpm.labelCompare(ver, cur_ver) == cmp_res: - if entry.name not in candid \ - or rpm.labelCompare(candid[entry.name][0], ver) < 0: - insert = True - if insert: - candid[entry.name] = DlEntry(ver, entry.reponame, entry.dlsize) - + query_res = filter_version(query_res, app, version_selector) # XXX: As it's possible to include version information in `template`, # perhaps the messages can be improved - if len(candid) == 0: + if len(query_res) == 0: if version_selector == VersionSelector.LATEST: parser.error('Template \'%s\' not found.' % template) elif version_selector == VersionSelector.REINSTALL: @@ -707,10 +727,12 @@ def get_dl_list( file=sys.stderr) # Merge & choose the template with the highest version - for name, dlentry in candid.items(): - if name not in full_candid \ - or rpm.labelCompare(full_candid[name].evr, dlentry.evr) < 0: - full_candid[name] = dlentry + for entry in query_res: + if entry.name not in full_candid \ + or rpm.labelCompare(full_candid[entry.name].evr, + entry.evr) < 0: + full_candid[entry.name] = \ + DlEntry(entry.evr, entry.reponame, entry.dlsize) return full_candid From 4f9757ca88633671fd328d582c516e0f99747bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 6 Feb 2021 03:36:06 +0100 Subject: [PATCH 097/119] qvm-template: by default list only latest available template But add --all-versions option to get all the available versions. --- qubesadmin/tests/tools/qvm_template.py | 81 +++++++++++++++++++++++++- qubesadmin/tools/qvm_template.py | 4 ++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 348b39c..6a5d6e1 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -2088,6 +2088,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 available=False, extras=False, upgrades=False, + all_versions=True, machine_readable=False, machine_readable_json=False, templates=['test-vm*'] @@ -2150,6 +2151,7 @@ qubes-template-fedora-32|0|4.1|20200101|qubes-templates-itl|1048576|2020-01-23 0 available=True, extras=False, upgrades=False, + all_versions=True, machine_readable=False, machine_readable_json=False, templates=['fedora-32', 'fedora-31'] @@ -2179,6 +2181,19 @@ f'''Available Templates @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') def test_151_list_templates_available_all_success(self, mock_query): mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'fedora-31', + '1', + '4.1', + '20190101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2019, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), qubesadmin.tools.qvm_template.Template( 'fedora-31', '1', @@ -2191,7 +2206,7 @@ f'''Available Templates 'https://qubes-os.org', 'Qubes template for fedora-31', 'Qubes template\n for fedora-31\n' - ) + ), ] args = argparse.Namespace( all=False, @@ -2199,6 +2214,60 @@ f'''Available Templates available=True, extras=False, upgrades=False, + all_versions=True, + machine_readable=False, + machine_readable_json=False, + templates=[] + ) + with mock.patch('sys.stdout', new=io.StringIO()) as mock_out: + qubesadmin.tools.qvm_template.list_templates( + args, self.app, 'list') + self.assertEqual(mock_out.getvalue(), +'''Available Templates +[('fedora-31', '1:4.1-20190101', 'qubes-templates-itl'), ('fedora-31', '1:4.1-20200101', 'qubes-templates-itl')] +''') + self.assertEqual(mock_query.mock_calls, [ + mock.call(args, self.app) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_template.qrexec_repoquery') + def test_151_list_templates_available_only_latest_success(self, mock_query): + mock_query.return_value = [ + qubesadmin.tools.qvm_template.Template( + 'fedora-31', + '1', + '4.1', + '20190101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2019, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-31', + '1', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), + ] + args = argparse.Namespace( + all=False, + installed=False, + available=True, + extras=False, + upgrades=False, + all_versions=False, machine_readable=False, machine_readable_json=False, templates=[] @@ -2283,6 +2352,7 @@ f'''Available Templates available=False, extras=True, upgrades=False, + all_versions=True, machine_readable=False, machine_readable_json=False, templates=['test-vm*'] @@ -2385,6 +2455,7 @@ f'''Available Templates available=False, extras=False, upgrades=True, + all_versions=True, machine_readable=False, machine_readable_json=False, templates=['test-vm*'] @@ -2463,6 +2534,7 @@ f'''Available Templates available=False, extras=False, upgrades=False, + all_versions=True, machine_readable=False, machine_readable_json=False, templates=['test-vm*'] @@ -2482,6 +2554,7 @@ Available Templates available=False, extras=False, upgrades=False, + all_versions=True, machine_readable=False, machine_readable_json=False, templates=['test-vm*'] @@ -2501,6 +2574,7 @@ Available Templates available=False, extras=False, upgrades=False, + all_versions=True, machine_readable=False, machine_readable_json=False, templates=['test-vm*'] @@ -2520,6 +2594,7 @@ Available Templates available=False, extras=False, upgrades=False, + all_versions=True, machine_readable=True, machine_readable_json=False, templates=['test-vm*'] @@ -2537,6 +2612,7 @@ available|test-vm|2:4.1-2020|qubes-templates-itl available=False, extras=False, upgrades=False, + all_versions=True, machine_readable=True, machine_readable_json=False, templates=['test-vm*'] @@ -2554,6 +2630,7 @@ available|test-vm|2|4.1|2020|qubes-templates-itl|1048576|2020-09-01 14:30:00||GP available=False, extras=False, upgrades=False, + all_versions=True, machine_readable=False, machine_readable_json=True, templates=['test-vm*'] @@ -2570,6 +2647,7 @@ available|test-vm|2|4.1|2020|qubes-templates-itl|1048576|2020-09-01 14:30:00||GP available=False, extras=False, upgrades=False, + all_versions=True, machine_readable=False, machine_readable_json=True, templates=['test-vm*'] @@ -2588,6 +2666,7 @@ r'''{"installed": [{"name": "test-vm-2", "epoch": "1", "version": "4.0", "releas available=True, extras=False, upgrades=False, + all_versions=True, machine_readable=False, machine_readable_json=False, templates=[] diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 0daeab9..b15292e 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -176,6 +176,8 @@ def get_parser() -> argparse.ArgumentParser: ' locally but not in repos) templates.')) parser_x.add_argument('--upgrades', action='store_true', help='Show available upgrades.') + parser_x.add_argument('--all-versions', action='store_true', + help='Show all available versions, not only the latest.') readable = parser_x.add_mutually_exclusive_group() readable.add_argument('--machine-readable', action='store_true', help='Enable machine-readable output.') @@ -1128,6 +1130,8 @@ def list_templates(args: argparse.Namespace, query_res = list(query_res_set) else: query_res = qrexec_repoquery(args, app) + if not args.all_versions: + query_res = filter_version(query_res, app) if args.installed or args.all: for vm in app.domains: From b7446afe3baaf1093a229ecdcef56df7cc305476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 6 Feb 2021 04:49:54 +0100 Subject: [PATCH 098/119] qvm-template: use key specified in the repo definition if possible This makes the package verified against _only_ the key specified in the repo config, not all the trusted keys. If repo does not specify a key, use the default one (change this to a single file, instead of the whole directory). Existing 'gpgkey' entry pointing at non-existing file will result in an error. --- qubesadmin/tests/tools/qvm_template.py | 14 +++++++- qubesadmin/tools/qvm_template.py | 49 ++++++++++++++++++-------- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 6a5d6e1..c131a4b 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -152,7 +152,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) self.assertAllCalled() - @mock.patch('qubesadmin.tools.qvm_template.get_keys') + @mock.patch('qubesadmin.tools.qvm_template.get_keys_for_repos') def test_090_install_lock(self, mock_get_keys): class SuccessError(Exception): pass @@ -251,6 +251,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): keyring='/tmp', nogpgcheck=False, cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', yes=False, allow_pv=False, pool=None @@ -357,6 +359,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): keyring='/tmp', nogpgcheck=False, cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', yes=False, allow_pv=True, pool='my-pool' @@ -432,6 +436,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): keyring='/tmp', nogpgcheck=False, cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', yes=False, allow_pv=False, pool=None @@ -496,6 +502,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): keyring='/tmp', nogpgcheck=False, cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', yes=False, allow_pv=False, pool=None @@ -563,6 +571,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): keyring='/tmp', nogpgcheck=False, cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', yes=False, allow_pv=False, pool=None @@ -611,6 +621,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): keyring='/tmp', nogpgcheck=False, cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', yes=False, allow_pv=False, pool=None diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index b15292e..a9871d5 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -21,6 +21,7 @@ import argparse import collections +import configparser import datetime import enum import fcntl @@ -104,8 +105,10 @@ def get_parser() -> argparse.ArgumentParser: help=('Specify files containing DNF repository configuration.' ' Can be used more than once.')) parser_main.add_argument('--keyring', - default='/etc/qubes/repo-templates/keys', - help='Specify directory containing RPM public keys.') + default='/etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-4.1-primary', + help='Specify a file containing default RPM public key. ' + 'Individual repositories may point at repo-specific key ' + 'using \'gpgkey\' option') parser_main.add_argument('--updatevm', default=UPDATEVM, help=('Specify VM to download updates from.' ' (Set to empty string to specify the current VM.)')) @@ -573,18 +576,32 @@ def qrexec_download( raise ConnectionError( "qrexec call 'qubes.TemplateDownload' failed.") -def get_keys(key_dir: str) -> typing.List[str]: - """List gpg keys""" - keys = [] - for name in os.listdir(key_dir): - path = os.path.join(key_dir, name) - if os.path.isfile(path): - keys.append(path) + +def get_keys_for_repos(repo_files: typing.List[str], + releasever: str) -> typing.Dict[str, str]: + """List gpg keys + + Returns a dict reponame -> key path + """ + keys = {} + for repo_file in repo_files: + repo_config = configparser.ConfigParser() + repo_config.read(repo_file) + for repo in repo_config.sections(): + try: + gpgkey_url = repo_config.get(repo, 'gpgkey') + except configparser.NoOptionError: + continue + gpgkey_url = gpgkey_url.replace('$releasever', releasever) + # support only file:// urls + if gpgkey_url.startswith('file://'): + keys[repo] = gpgkey_url[len('file://'):] return keys + def verify_rpm( path: str, - keys: typing.List[str], + key: str, nogpgcheck: bool = False ) -> rpm.hdr: """Verify the digest and signature of a RPM package and return the package @@ -602,9 +619,8 @@ def verify_rpm( """ if not nogpgcheck: with tempfile.TemporaryDirectory() as rpmdb_dir: - for key in keys: - subprocess.check_call( - ['rpmkeys', '--dbpath=' + rpmdb_dir, '--import', key]) + subprocess.check_call( + ['rpmkeys', '--dbpath=' + rpmdb_dir, '--import', key]) try: output = subprocess.check_output( ['rpmkeys', '--dbpath=' + rpmdb_dir, '--checksig', path]) @@ -834,7 +850,7 @@ def install( :param override_existing: Whether to override existing packages. Used for reinstall, upgrade, and downgrade operations """ - keys = get_keys(args.keyring) + keys = get_keys_for_repos(args.repo_files, args.releasever) unverified_rpm_list = [] # rpmfile, reponame verified_rpm_list = [] @@ -847,7 +863,10 @@ def install( else: path = rpmfile - package_hdr = verify_rpm(path, keys, args.nogpgcheck) + repo_key = keys.get(reponame) + if repo_key is None: + repo_key = args.keyring + package_hdr = verify_rpm(path, repo_key, args.nogpgcheck) if not package_hdr: parser.error('Package \'%s\' verification failed.' % rpmfile) From 8795668233faad672cd7b12f6313ad7f20b8d02c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 6 Feb 2021 07:30:28 +0100 Subject: [PATCH 099/119] qvm-template-postprocess: do not generate appmenus twice Skip initial generate, as it's done before actual menu entries are extracted from the template. But do call it if we aren't going to extract menu entries initially - it will create just "settings" menu entry. --- .../tests/tools/qvm_template_postprocess.py | 12 ++++++------ qubesadmin/tools/qvm_template_postprocess.py | 17 ++++++++++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template_postprocess.py b/qubesadmin/tests/tools/qvm_template_postprocess.py index 1a77af0..fded880 100644 --- a/qubesadmin/tests/tools/qvm_template_postprocess.py +++ b/qubesadmin/tests/tools/qvm_template_postprocess.py @@ -220,7 +220,7 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase): vm = self.app.domains['test-vm'] with mock.patch('subprocess.check_call') as mock_proc: qubesadmin.tools.qvm_template_postprocess.import_appmenus( - vm, self.source_dir.name) + vm, self.source_dir.name, skip_generate=False) self.assertEqual(mock_proc.mock_calls, [ mock.call(['qvm-appmenus', '--set-default-whitelist=' + os.path.join(self.source_dir.name, @@ -282,7 +282,7 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase): vm = self.app.domains['test-vm'] with mock.patch('subprocess.check_call') as mock_proc: qubesadmin.tools.qvm_template_postprocess.import_appmenus( - vm, self.source_dir.name) + vm, self.source_dir.name, skip_generate=False) self.assertEqual(mock_proc.mock_calls, [ mock.call(['runuser', '-u', 'user', '--', 'env', 'DISPLAY=:0', 'qvm-appmenus', @@ -316,7 +316,7 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase): vm = self.app.domains['test-vm'] with mock.patch('subprocess.check_call') as mock_proc: qubesadmin.tools.qvm_template_postprocess.import_appmenus( - vm, self.source_dir.name) + vm, self.source_dir.name, skip_generate=False) self.assertEqual(mock_proc.mock_calls, []) self.assertAllCalled() @@ -373,7 +373,7 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase): mock_import_root_img.assert_called_once_with(self.app.domains[ 'test-vm'], self.source_dir.name) mock_import_appmenus.assert_called_once_with(self.app.domains[ - 'test-vm'], self.source_dir.name) + 'test-vm'], self.source_dir.name, skip_generate=True) if qubesadmin.tools.qvm_template_postprocess.have_events: mock_domain_shutdown.assert_called_once_with([self.app.domains[ 'test-vm']]) @@ -428,7 +428,7 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase): mock_reset_private_img.assert_called_once_with(self.app.domains[ 'test-vm']) mock_import_appmenus.assert_called_once_with(self.app.domains[ - 'test-vm'], self.source_dir.name) + 'test-vm'], self.source_dir.name, skip_generate=True) if qubesadmin.tools.qvm_template_postprocess.have_events: mock_domain_shutdown.assert_called_once_with([self.app.domains[ 'test-vm']]) @@ -469,7 +469,7 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase): mock_reset_private_img.assert_called_once_with(self.app.domains[ 'test-vm']) mock_import_appmenus.assert_called_once_with(self.app.domains[ - 'test-vm'], self.source_dir.name) + 'test-vm'], self.source_dir.name, skip_generate=False) if qubesadmin.tools.qvm_template_postprocess.have_events: self.assertFalse(mock_domain_shutdown.called) self.assertEqual(self.app.service_calls, []) diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py index b5e04c3..3b1976a 100644 --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -143,8 +143,14 @@ def reset_private_img(vm): vm.volumes['private'].clear_data() -def import_appmenus(vm, source_dir): - '''Import appmenus settings into VM object (later: GUI VM)''' +def import_appmenus(vm, source_dir, skip_generate=True): + """Import appmenus settings into VM object (later: GUI VM) + + :param vm: QubesVM object of just imported template + :param source_dir: directory with source files + :param skip_generate: do not generate actual menu entries, + only set item lists + """ if os.getuid() == 0: try: qubes_group = grp.getgrnam('qubes') @@ -171,6 +177,9 @@ def import_appmenus(vm, source_dir): as fd: vm.features['netvm-menu-items'] = ' '.join([x.rstrip() for x in fd]) + if skip_generate: + return + # TODO: change this to qrexec calls to GUI VM, when GUI VM will be # implemented try: @@ -283,7 +292,9 @@ def post_install(args): vm.log.info('Clearing private volume') reset_private_img(vm) vm.installed_by_rpm = not args.no_installed_by_rpm - import_appmenus(vm, args.dir) + # do not generate actual menu entries, if post-install service will be + # executed anyway + import_appmenus(vm, args.dir, skip_generate=not args.skip_start) conf_path = os.path.join(args.dir, 'template.conf') if os.path.exists(conf_path): From ce36dc55c590cc59a6e93513ca6fffd97568ab4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 6 Feb 2021 07:31:37 +0100 Subject: [PATCH 100/119] qvm-template: improve error reporting Do not print the whole traceback by default - do that only when --verbose is used. --- qubesadmin/tools/qvm_template.py | 67 +++++++++++++++++--------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index a9871d5..df9c5ab 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -1477,38 +1477,43 @@ def main(args: typing.Optional[typing.Sequence[str]] = None, if p_args.updatevm is UPDATEVM: p_args.updatevm = app.updatevm - if p_args.refresh: - qrexec_repoquery(p_args, app, refresh=True) + try: + if p_args.refresh: + qrexec_repoquery(p_args, app, refresh=True) - if p_args.command == 'download': - download(p_args, app) - elif p_args.command == 'install': - install(p_args, app) - elif p_args.command == 'reinstall': - install(p_args, app, version_selector=VersionSelector.REINSTALL, - override_existing=True) - elif p_args.command == 'downgrade': - install(p_args, app, version_selector=VersionSelector.LATEST_LOWER, - override_existing=True) - elif p_args.command == 'upgrade': - install(p_args, app, version_selector=VersionSelector.LATEST_HIGHER, - override_existing=True) - elif p_args.command == 'list': - list_templates(p_args, app, 'list') - elif p_args.command == 'info': - list_templates(p_args, app, 'info') - elif p_args.command == 'search': - search(p_args, app) - elif p_args.command == 'remove': - remove(p_args, app, disassoc=p_args.disassoc) - elif p_args.command == 'purge': - remove(p_args, app, purge=True) - elif p_args.command == 'clean': - clean(p_args, app) - elif p_args.command == 'repolist': - repolist(p_args, app) - else: - parser.error('Command \'%s\' not supported.' % p_args.command) + if p_args.command == 'download': + download(p_args, app) + elif p_args.command == 'install': + install(p_args, app) + elif p_args.command == 'reinstall': + install(p_args, app, version_selector=VersionSelector.REINSTALL, + override_existing=True) + elif p_args.command == 'downgrade': + install(p_args, app, version_selector=VersionSelector.LATEST_LOWER, + override_existing=True) + elif p_args.command == 'upgrade': + install(p_args, app, version_selector=VersionSelector.LATEST_HIGHER, + override_existing=True) + elif p_args.command == 'list': + list_templates(p_args, app, 'list') + elif p_args.command == 'info': + list_templates(p_args, app, 'info') + elif p_args.command == 'search': + search(p_args, app) + elif p_args.command == 'remove': + remove(p_args, app, disassoc=p_args.disassoc) + elif p_args.command == 'purge': + remove(p_args, app, purge=True) + elif p_args.command == 'clean': + clean(p_args, app) + elif p_args.command == 'repolist': + repolist(p_args, app) + else: + parser.error('Command \'%s\' not supported.' % p_args.command) + except Exception as e: # pylint: disable=broad-except + print('ERROR: ' + str(e), file=sys.stderr) + app.log.debug(str(e), exc_info=sys.exc_info()) + return 1 return 0 From e424c7df9c402d8a730052efe7805d84a0becb77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 18 Feb 2021 20:03:50 +0100 Subject: [PATCH 101/119] qvm-template: verify template package signature directly at download Make the download() function save the package into a temporary space and move to the target location only after checking the signature. This is safer option than requiring all callers to explicitly verify the signature. Also, make the download() function verify if the template name inside the package matches what was requested. Especially, make `qvm-template download` action verify the signature too. On `qvm-template install` avoid checking the signature again for downloaded packages, by passing extra argument to the verify_rpm() function. But still verify signature of packages loaded from disk. --- qubesadmin/tests/tools/qvm_template.py | 391 ++++++++++++++++++++----- qubesadmin/tools/qvm_template.py | 123 ++++---- 2 files changed, 384 insertions(+), 130 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index c131a4b..2746028 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -1,3 +1,4 @@ +import re from unittest import mock import argparse import asyncio @@ -14,6 +15,13 @@ import rpm import qubesadmin.tests import qubesadmin.tools.qvm_template +class re_str(str): + def __eq__(self, other): + return bool(re.match(self, other)) + + def __hash__(self): + return super().__hash__() + class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): def setUp(self): # Print str(list) directly so that the output is consistent no matter @@ -248,7 +256,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): path = template_file.name args = argparse.Namespace( templates=[path], - keyring='/tmp', + keyring='/tmp/keyring.gpg', nogpgcheck=False, cachedir='/var/cache/qvm-template', repo_files=[], @@ -267,8 +275,9 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # Nothing downloaded mock_dl.assert_called_with(args, self.app, - path_override='/var/tmp/qvm-template-tmpdir', - dl_list={}, suffix='.unverified', version_selector=selector) + dl_list={}, version_selector=selector) + mock_verify.assert_called_once_with(template_file.name, '/tmp/keyring.gpg', + nogpgcheck=False) # Package is extracted mock_extract.assert_called_with('test-vm', path, '/var/tmp/qvm-template-tmpdir') @@ -375,8 +384,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # Nothing downloaded mock_dl.assert_called_with(args, self.app, - path_override='/var/tmp/qvm-template-tmpdir', - dl_list={}, suffix='.unverified', version_selector=selector) + dl_list={}, version_selector=selector) # Package is extracted mock_extract.assert_called_with('test-vm', path, '/var/tmp/qvm-template-tmpdir') @@ -520,8 +528,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # Nothing downloaded self.assertEqual(mock_dl.mock_calls, [ - mock.call(args, self.app, path_override='/var/tmp/qvm-template-tmpdir', - dl_list={}, suffix='.unverified', version_selector=selector) + mock.call(args, self.app, + dl_list={}, version_selector=selector) ]) # Should not be executed: self.assertEqual(mock_extract.mock_calls, []) @@ -644,6 +652,119 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): self.assertEqual(mock_rename.mock_calls, []) self.assertAllCalled() + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + def test_107_install_download_success( + self, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = b'0\0' + build_time = '2020-09-01 14:30:00' # 1598970600 + install_time = '2020-09-01 15:30:00' + for key, val in [ + ('name', 'test-vm'), + ('epoch', '2'), + ('version', '4.1'), + ('release', '2020'), + ('reponame', 'qubes-templates-itl'), + ('buildtime', build_time), + ('installtime', install_time), + ('license', 'GPL'), + ('url', 'https://qubes-os.org'), + ('summary', 'Summary'), + ('description', 'Desc|desc')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Set', + f'template-{key}', + val.encode())] = b'0\0' + mock_dl.return_value = {'test-vm': { + rpm.RPMTAG_NAME : 'qubes-template-test-vm', + rpm.RPMTAG_BUILDTIME : 1598970600, + rpm.RPMTAG_DESCRIPTION : 'Desc\ndesc', + rpm.RPMTAG_EPOCHNUM : 2, + rpm.RPMTAG_LICENSE : 'GPL', + rpm.RPMTAG_RELEASE : '2020', + rpm.RPMTAG_SUMMARY : 'Summary', + rpm.RPMTAG_URL : 'https://qubes-os.org', + rpm.RPMTAG_VERSION : '4.1' + }} + dl_list = { + 'test-vm': qubesadmin.tools.qvm_template.DlEntry( + ('1', '4.1', '20200101'), 'qubes-templates-itl', 1048576) + } + mock_dl_list.return_value = dl_list + mock_call.side_effect = self.add_new_vm_side_effect + mock_time = mock.Mock(wraps=datetime.datetime) + mock_time.now.return_value = \ + datetime.datetime(2020, 9, 1, 15, 30, tzinfo=datetime.timezone.utc) + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ + mock.patch('datetime.datetime', new=mock_time), \ + mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + args = argparse.Namespace( + templates='test-vm', + keyring='/tmp/keyring.gpg', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', + yes=False, + keep_cache=True, + allow_pv=False, + pool=None + ) + mock_tmpdir.return_value.__enter__.return_value = \ + '/var/tmp/qvm-template-tmpdir' + qubesadmin.tools.qvm_template.install(args, self.app) + # Attempt to get download list + selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST + self.assertEqual(mock_dl_list.mock_calls, [ + mock.call(args, self.app, version_selector=selector) + ]) + # Nothing downloaded + mock_dl.assert_called_with(args, self.app, + dl_list=dl_list, version_selector=selector) + # download already verify the package internally + self.assertEqual(mock_verify.mock_calls, []) + # Package is extracted + mock_extract.assert_called_with('test-vm', + '/var/cache/qvm-template/qubes-template-test-vm-1:4.1-20200101.rpm', + '/var/tmp/qvm-template-tmpdir') + # No packages overwritten, so no confirm needed + self.assertEqual(mock_confirm.mock_calls, []) + # qvm-template-postprocess is called + self.assertEqual(mock_call.mock_calls, [ + mock.call([ + 'qvm-template-postprocess', + '--really', + '--no-installed-by-rpm', + 'post-install', + 'test-vm', + '/var/tmp/qvm-template-tmpdir' + '/var/lib/qubes/vm-templates/test-vm' + ]) + ]) + # Cache directory created + self.assertEqual(mock_mkdirs.mock_calls, [ + mock.call(args.cachedir, exist_ok=True) + ]) + # No templates downloaded, thus no renames needed + self.assertEqual(mock_rename.mock_calls, []) + self.assertAllCalled() + def test_110_qrexec_payload_refresh_success(self): with tempfile.NamedTemporaryFile() as repo_conf1, \ tempfile.NamedTemporaryFile() as repo_conf2: @@ -3188,11 +3309,27 @@ test-vm : Qubes template for fedora-31 ]) self.assertAllCalled() + def _mock_qrexec_download(self, args, app, spec, path, + dlsize=None, refresh=False): + self.assertFalse(os.path.exists(path), + '{} should not exist before'.format(path)) + # just create an empty file + with open(path, 'wb') as f: + if f is not None: + f.truncate(dlsize) + + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') - def test_180_download_success(self, mock_qrexec, mock_dllist): + def test_180_download_success(self, mock_qrexec, mock_dllist, + mock_verify_rpm): + mock_qrexec.side_effect = self._mock_qrexec_download with tempfile.TemporaryDirectory() as dir: args = argparse.Namespace( + repo_files=[], + keyring='/tmp/keyring.gpg', + releasever='4.1', + nogpgcheck=False, retries=1 ) qubesadmin.tools.qvm_template.download(args, self.app, dir, { @@ -3202,25 +3339,36 @@ test-vm : Qubes template for fedora-31 ('0', '1', '2'), 'qubes-templates-itl-testing', 2048576) - }, '.unverified') + }) self.assertEqual(mock_qrexec.mock_calls, [ mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', - dir + '/qubes-template-fedora-31-1:2-3.rpm.unverified', + re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), 1048576), mock.call(args, self.app, 'qubes-template-fedora-32-0:1-2', - dir + '/qubes-template-fedora-32-0:1-2.rpm.unverified', + re_str(dir + '/.*/qubes-template-fedora-32-0:1-2.rpm.UNTRUSTED'), 2048576) ]) self.assertEqual(mock_dllist.mock_calls, []) - self.assertTrue(all( - [x.endswith('.unverified') for x in os.listdir(dir)])) + self.assertEqual(mock_verify_rpm.mock_calls, [ + mock.call(re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), + '/tmp/keyring.gpg', template_name='fedora-31'), + mock.call(re_str(dir + '/.*/qubes-template-fedora-32-0:1-2.rpm.UNTRUSTED'), + '/tmp/keyring.gpg', template_name='fedora-32'), + ]) + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') - def test_181_download_success_nosuffix(self, mock_qrexec, mock_dllist): + def test_181_download_success_nosuffix(self, mock_qrexec, mock_dllist, + mock_verify_rpm): + mock_qrexec.side_effect = self._mock_qrexec_download with tempfile.TemporaryDirectory() as dir: args = argparse.Namespace( retries=1, + repo_files=[], + keyring='/tmp/keyring.gpg', + releasever='4.1', + nogpgcheck=False, downloaddir=dir ) with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: @@ -3230,28 +3378,39 @@ test-vm : Qubes template for fedora-31 }) self.assertEqual(mock_qrexec.mock_calls, [ mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', - dir + '/qubes-template-fedora-31-1:2-3.rpm', + re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), 1048576) ]) self.assertEqual(mock_dllist.mock_calls, []) + self.assertEqual(mock_verify_rpm.mock_calls, [ + mock.call(re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), + '/tmp/keyring.gpg', template_name='fedora-31'), + ]) + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') - def test_182_download_success_getdllist(self, mock_qrexec, mock_dllist): + def test_182_download_success_getdllist(self, mock_qrexec, mock_dllist, + mock_verify_rpm): + mock_qrexec.side_effect = self._mock_qrexec_download mock_dllist.return_value = { 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( ('1', '2', '3'), 'qubes-templates-itl', 1048576) } with tempfile.TemporaryDirectory() as dir: args = argparse.Namespace( - retries=1 + retries=1, + repo_files=[], + keyring='/tmp/keyring.gpg', + releasever='4.1', + nogpgcheck=False, ) qubesadmin.tools.qvm_template.download(args, self.app, - dir, None, '.unverified', + dir, None, qubesadmin.tools.qvm_template.VersionSelector.LATEST_LOWER) self.assertEqual(mock_qrexec.mock_calls, [ mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', - dir + '/qubes-template-fedora-31-1:2-3.rpm.unverified', + re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), 1048576) ]) self.assertEqual(mock_dllist.mock_calls, [ @@ -3260,40 +3419,54 @@ test-vm : Qubes template for fedora-31 qubesadmin.tools.qvm_template.\ VersionSelector.LATEST_LOWER) ]) - self.assertTrue(all( - [x.endswith('.unverified') for x in os.listdir(dir)])) + self.assertEqual(mock_verify_rpm.mock_calls, [ + mock.call(re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), + '/tmp/keyring.gpg', template_name='fedora-31'), + ]) + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') - def test_183_download_success_downloaddir(self, mock_qrexec, mock_dllist): + def test_183_download_success_downloaddir(self, mock_qrexec, mock_dllist, + mock_verify_rpm): + mock_qrexec.side_effect = self._mock_qrexec_download with tempfile.TemporaryDirectory() as dir: args = argparse.Namespace( retries=1, + repo_files=[], + keyring='/tmp/keyring.gpg', + releasever='4.1', + nogpgcheck=False, downloaddir=dir ) qubesadmin.tools.qvm_template.download(args, self.app, None, { 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( ('1', '2', '3'), 'qubes-templates-itl', 1048576) - }, '.unverified') + }) self.assertEqual(mock_qrexec.mock_calls, [ mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', - dir + '/qubes-template-fedora-31-1:2-3.rpm.unverified', + re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), 1048576) ]) self.assertEqual(mock_dllist.mock_calls, []) - self.assertTrue(all( - [x.endswith('.unverified') for x in os.listdir(dir)])) + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') - def test_184_download_success_exists(self, mock_qrexec, mock_dllist): + def test_184_download_success_exists(self, mock_qrexec, mock_dllist, + mock_verify_rpm): + mock_qrexec.side_effect = self._mock_qrexec_download with tempfile.TemporaryDirectory() as dir: with open(os.path.join( - dir, 'qubes-template-fedora-31-1:2-3.rpm.unverified'), + dir, 'qubes-template-fedora-31-1:2-3.rpm'), 'w') as _: pass args = argparse.Namespace( retries=1, + repo_files=[], + keyring='/tmp/keyring.gpg', + releasever='4.1', + nogpgcheck=False, downloaddir=dir ) with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: @@ -3304,47 +3477,22 @@ test-vm : Qubes template for fedora-31 ('0', '1', '2'), 'qubes-templates-itl-testing', 2048576) - }, '.unverified') + }) self.assertTrue('already exists, skipping' in mock_err.getvalue()) self.assertEqual(mock_qrexec.mock_calls, [ mock.call(args, self.app, 'qubes-template-fedora-32-0:1-2', - dir + '/qubes-template-fedora-32-0:1-2.rpm.unverified', + re_str(dir + '/.*/qubes-template-fedora-32-0:1-2.rpm.UNTRUSTED'), 2048576) ]) self.assertEqual(mock_dllist.mock_calls, []) - self.assertTrue(all( - [x.endswith('.unverified') for x in os.listdir(dir)])) + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') - def test_185_download_success_existsmove(self, mock_qrexec, mock_dllist): - with tempfile.TemporaryDirectory() as dir: - with open(os.path.join( - dir, 'qubes-template-fedora-31-1:2-3.rpm'), - 'w') as _: - pass - args = argparse.Namespace( - retries=1, - downloaddir=dir - ) - with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: - qubesadmin.tools.qvm_template.download(args, self.app, None, { - 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( - ('1', '2', '3'), 'qubes-templates-itl', 1048576) - }, '.unverified') - self.assertTrue('already exists, skipping' - in mock_err.getvalue()) - self.assertEqual(mock_qrexec.mock_calls, []) - self.assertEqual(mock_dllist.mock_calls, []) - self.assertTrue(os.path.exists( - dir + '/qubes-template-fedora-31-1:2-3.rpm.unverified')) - self.assertTrue(all( - [x.endswith('.unverified') for x in os.listdir(dir)])) - - @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') - @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') - def test_186_download_success_existsnosuffix(self, mock_qrexec, mock_dllist): + def test_185_download_success_existsmove(self, mock_qrexec, mock_dllist, + mock_verify_rpm): + mock_qrexec.side_effect = self._mock_qrexec_download with tempfile.TemporaryDirectory() as dir: with open(os.path.join( dir, 'qubes-template-fedora-31-1:2-3.rpm'), @@ -3352,6 +3500,10 @@ test-vm : Qubes template for fedora-31 pass args = argparse.Namespace( retries=1, + repo_files=[], + keyring='/tmp/keyring.gpg', + releasever='4.1', + nogpgcheck=False, downloaddir=dir ) with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: @@ -3366,19 +3518,57 @@ test-vm : Qubes template for fedora-31 self.assertTrue(os.path.exists( dir + '/qubes-template-fedora-31-1:2-3.rpm')) + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') - def test_187_download_success_retry(self, mock_qrexec, mock_dllist): + def test_186_download_success_existsnosuffix(self, mock_qrexec, mock_dllist, + mock_verify_rpm): + mock_qrexec.side_effect = self._mock_qrexec_download + with tempfile.TemporaryDirectory() as dir: + with open(os.path.join( + dir, 'qubes-template-fedora-31-1:2-3.rpm'), + 'w') as _: + pass + args = argparse.Namespace( + retries=1, + repo_files=[], + keyring='/tmp/keyring.gpg', + releasever='4.1', + nogpgcheck=False, + downloaddir=dir + ) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + qubesadmin.tools.qvm_template.download(args, self.app, None, { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576) + }) + self.assertTrue('already exists, skipping' + in mock_err.getvalue()) + self.assertEqual(mock_qrexec.mock_calls, []) + self.assertEqual(mock_dllist.mock_calls, []) + self.assertTrue(os.path.exists( + dir + '/qubes-template-fedora-31-1:2-3.rpm')) + + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_187_download_success_retry(self, mock_qrexec, mock_dllist, + mock_verify_rpm): counter = 0 - def f(*args): + def f(*args, **kwargs): nonlocal counter counter += 1 if counter == 1: raise ConnectionError + self._mock_qrexec_download(*args, **kwargs) mock_qrexec.side_effect = f with tempfile.TemporaryDirectory() as dir: args = argparse.Namespace( retries=2, + repo_files=[], + keyring='/tmp/keyring.gpg', + releasever='4.1', + nogpgcheck=False, downloaddir=dir ) with mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ @@ -3389,31 +3579,39 @@ test-vm : Qubes template for fedora-31 }) self.assertTrue('retrying...' in mock_err.getvalue()) self.assertEqual(mock_rm.mock_calls, [ - mock.call(dir + '/qubes-template-fedora-31-1:2-3.rpm') + mock.call(re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED')) ]) self.assertEqual(mock_qrexec.mock_calls, [ mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', - dir + '/qubes-template-fedora-31-1:2-3.rpm', + re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), 1048576), mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', - dir + '/qubes-template-fedora-31-1:2-3.rpm', + re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), 1048576) ]) self.assertEqual(mock_dllist.mock_calls, []) + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') - def test_188_download_fail_retry(self, mock_qrexec, mock_dllist): + def test_188_download_fail_retry(self, mock_qrexec, mock_dllist, + mock_verify_rpm): + mock_qrexec.side_effect = self._mock_qrexec_download counter = 0 - def f(*args): + def f(*args, **kwargs): nonlocal counter counter += 1 if counter <= 3: raise ConnectionError + self._mock_qrexec_download(*args, **kwargs) mock_qrexec.side_effect = f with tempfile.TemporaryDirectory() as dir: args = argparse.Namespace( retries=3, + repo_files=[], + keyring='/tmp/keyring.gpg', + releasever='4.1', + nogpgcheck=False, downloaddir=dir ) with mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ @@ -3427,32 +3625,38 @@ test-vm : Qubes template for fedora-31 self.assertEqual(mock_err.getvalue().count('retrying...'), 2) self.assertTrue('download failed' in mock_err.getvalue()) self.assertEqual(mock_rm.mock_calls, [ - mock.call(dir + '/qubes-template-fedora-31-1:2-3.rpm'), - mock.call(dir + '/qubes-template-fedora-31-1:2-3.rpm'), - mock.call(dir + '/qubes-template-fedora-31-1:2-3.rpm') + mock.call(re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED')), + mock.call(re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED')), + mock.call(re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED')) ]) self.assertEqual(mock_qrexec.mock_calls, [ mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', - dir + '/qubes-template-fedora-31-1:2-3.rpm', + re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), 1048576), mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', - dir + '/qubes-template-fedora-31-1:2-3.rpm', + re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), 1048576), mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', - dir + '/qubes-template-fedora-31-1:2-3.rpm', + re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), 1048576) ]) self.assertEqual(mock_dllist.mock_calls, []) + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') - def test_189_download_fail_interrupt(self, mock_qrexec, mock_dllist): + def test_189_download_fail_interrupt(self, mock_qrexec, mock_dllist, + mock_verify_rpm): def f(*args): raise RuntimeError mock_qrexec.side_effect = f with tempfile.TemporaryDirectory() as dir: args = argparse.Namespace( retries=3, + repo_files=[], + keyring='/tmp/keyring.gpg', + releasever='4.1', + nogpgcheck=False, downloaddir=dir ) with mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ @@ -3463,12 +3667,45 @@ test-vm : Qubes template for fedora-31 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( ('1', '2', '3'), 'qubes-templates-itl', 1048576) }) - self.assertEqual(mock_rm.mock_calls, [ - mock.call(dir + '/qubes-template-fedora-31-1:2-3.rpm') - ]) self.assertEqual(mock_qrexec.mock_calls, [ mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', - dir + '/qubes-template-fedora-31-1:2-3.rpm', + re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm'), 1048576) ]) self.assertEqual(mock_dllist.mock_calls, []) + + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_189_download_verify_fail(self, mock_qrexec, mock_dllist, + mock_verify_rpm): + mock_qrexec.side_effect = self._mock_qrexec_download + mock_verify_rpm.side_effect = \ + qubesadmin.tools.qvm_template.SignatureVerificationError + + with tempfile.TemporaryDirectory() as dir: + args = argparse.Namespace( + retries=3, + repo_files=[], + keyring='/tmp/keyring.gpg', + releasever='4.1', + nogpgcheck=False, + downloaddir=dir + ) + with self.assertRaises(qubesadmin.tools.qvm_template.SignatureVerificationError): + qubesadmin.tools.qvm_template.download( + args, self.app, None, { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576) + }) + self.assertEqual(mock_qrexec.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', + re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm'), + 1048576) + ]) + self.assertEqual(mock_dllist.mock_calls, []) + self.assertEqual(os.listdir(dir), []) + + # TODO: test unexpected name downloaded + # TODO: test truncated download + # TODO: test no disk space \ No newline at end of file diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index df9c5ab..406f839 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -146,8 +146,6 @@ def get_parser() -> argparse.ArgumentParser: help_str='Upgrade template packages.') for parser_x in [parser_install, parser_reinstall, parser_downgrade, parser_upgrade]: - parser_x.add_argument('--nogpgcheck', action='store_true', - help='Disable signature checks.') parser_x.add_argument('--allow-pv', action='store_true', help='Allow templates that set virt_mode to pv.') parser_x.add_argument('templates', nargs='*', metavar='TEMPLATESPEC') @@ -160,6 +158,8 @@ def get_parser() -> argparse.ArgumentParser: help='Specify download directory.') parser_x.add_argument('--retries', default=5, type=int, help='Specify maximum number of retries for downloads.') + parser_x.add_argument('--nogpgcheck', action='store_true', + help='Disable signature checks.') parser_download.add_argument('templates', nargs='*', metavar='TEMPLATESPEC') # qvm-template {list,info} @@ -602,7 +602,8 @@ def get_keys_for_repos(repo_files: typing.List[str], def verify_rpm( path: str, key: str, - nogpgcheck: bool = False + nogpgcheck: bool = False, + template_name: typing.Optional[str] = None ) -> rpm.hdr: """Verify the digest and signature of a RPM package and return the package header. @@ -614,6 +615,8 @@ def verify_rpm( :param path: Location of the RPM package :param nogpgcheck: Whether to allow invalid GPG signatures + :param template_name: expected template name - if specified, verifies if + the package name matches expected template name :return: RPM package header. If verification fails, raises an exception. """ @@ -635,6 +638,10 @@ def verify_rpm( tset = rpm.TransactionSet() tset.setVSFlags(rpm.RPMVSF_MASK_NOSIGNATURES) hdr = tset.hdrFromFdno(fd) + if template_name is not None: + if hdr[rpm.RPMTAG_NAME] != PACKAGE_NAME_PREFIX + template_name: + raise SignatureVerificationError( + 'Downloaded package does not match expected template name') return hdr def extract_rpm(name: str, path: str, target: str) -> bool: @@ -759,8 +766,8 @@ def download( app: qubesadmin.app.QubesBase, path_override: typing.Optional[str] = None, dl_list: typing.Optional[typing.Dict[str, DlEntry]] = None, - suffix: str = '', - version_selector: VersionSelector = VersionSelector.LATEST) -> None: + version_selector: VersionSelector = VersionSelector.LATEST) \ + -> typing.Dict[str, rpm.hdr]: """Command that downloads template packages. :param args: Arguments received by the application. @@ -770,50 +777,65 @@ def download( :param dl_list: Override list of templates to download. If not set or set to None, ``get_dl_list`` is called, which generates the list from ``args``. Optional - :param suffix: Suffix to add to the file name of downloaded packages. This - is useful if you want to distinguish between verified and unverified - packages. Defaults to an empty string :param version_selector: Specify algorithm to select the candidate version of a package. Defaults to ``VersionSelector.LATEST`` + :return package headers of downloaded templates """ if dl_list is None: dl_list = get_dl_list(args, app, version_selector=version_selector) + keys = get_keys_for_repos(args.repo_files, args.releasever) + + package_hdrs = {} + path = path_override if path_override is not None else args.downloaddir - for name, entry in dl_list.items(): - version_str = build_version_str(entry.evr) - spec = PACKAGE_NAME_PREFIX + name + '-' + version_str - target = os.path.join(path, '%s.rpm' % spec) - target_suffix = target + suffix - if os.path.exists(target_suffix): - print('\'%s\' already exists, skipping...' % target, - file=sys.stderr) - elif os.path.exists(target): - print('\'%s\' already exists, skipping...' % target, - file=sys.stderr) - os.rename(target, target_suffix) - else: + with tempfile.TemporaryDirectory(dir=path) as dl_dir: + for name, entry in dl_list.items(): + version_str = build_version_str(entry.evr) + spec = PACKAGE_NAME_PREFIX + name + '-' + version_str + target = os.path.join(path, '%s.rpm' % spec) + target_temp = os.path.join(dl_dir, '%s.rpm.UNTRUSTED' % spec) + repo_key = keys.get(entry.reponame) + if repo_key is None: + repo_key = args.keyring + if os.path.exists(target): + print('\'%s\' already exists, skipping...' % target, + file=sys.stderr) + # but still verify the package + verify_rpm(target, repo_key, template_name=name) + continue print('Downloading \'%s\'...' % spec, file=sys.stderr) done = False for attempt in range(args.retries): try: - qrexec_download(args, app, spec, target_suffix, + qrexec_download(args, app, spec, target_temp, entry.dlsize) + size = os.path.getsize(target_temp) + if size != entry.dlsize: + raise ConnectionError( + 'Downloaded file is {} bytes, expected {}'.format( + size, entry.dlsize)) done = True break except ConnectionError: - os.remove(target_suffix) + os.remove(target_temp) if attempt + 1 < args.retries: print('\'%s\' download failed, retrying...' % spec, file=sys.stderr) - except: - # Also remove file if interrupted by other means - os.remove(target_suffix) - raise if not done: print('Error: \'%s\' download failed.' % spec, file=sys.stderr) sys.exit(1) + if args.nogpgcheck: + print( + 'Warning: --nogpgcheck is ignored for downloaded templates', + file=sys.stderr) + package_hdr = verify_rpm(target_temp, repo_key, template_name=name) + # after package is verified, rename to the target location + os.rename(target_temp, target) + package_hdrs[name] = package_hdr + return package_hdrs + def locked(func): """Execute given function under a lock in *LOCK_FILE*""" @functools.wraps(func) @@ -854,21 +876,20 @@ def install( unverified_rpm_list = [] # rpmfile, reponame verified_rpm_list = [] - def verify(rpmfile, reponame, dl_dir=None): + def verify(rpmfile, reponame, package_hdr=None): """Verify package signature and version, remove "unverified" - suffix, and parse package header.""" - if dl_dir: - path = os.path.join( - dl_dir, os.path.basename(rpmfile) + UNVERIFIED_SUFFIX) - else: - path = rpmfile + suffix, and parse package header. - repo_key = keys.get(reponame) - if repo_key is None: - repo_key = args.keyring - package_hdr = verify_rpm(path, repo_key, args.nogpgcheck) - if not package_hdr: - parser.error('Package \'%s\' verification failed.' % rpmfile) + If package_hdr is provided, the signature check is skipped and only + other checks are performed.""" + if package_hdr is None: + repo_key = keys.get(reponame) + if repo_key is None: + repo_key = args.keyring + package_hdr = verify_rpm(rpmfile, repo_key, + nogpgcheck=args.nogpgcheck) + if not package_hdr: + parser.error('Package \'%s\' verification failed.' % rpmfile) package_name = package_hdr[rpm.RPMTAG_NAME] if not package_name.startswith(PACKAGE_NAME_PREFIX): @@ -877,9 +898,6 @@ def install( # Remove prefix to get the real template name name = package_name[len(PACKAGE_NAME_PREFIX):] - if path != rpmfile: - os.rename(path, rpmfile) - # Check if already installed if not override_existing and name in app.domains: print(('Template \'%s\' already installed, skipping...' @@ -927,7 +945,7 @@ def install( # First verify local RPMs and extract header for rpmfile, reponame in unverified_rpm_list: verify(rpmfile, reponame) - unverified_rpm_list = [] + unverified_rpm_list = {} os.makedirs(args.cachedir, exist_ok=True) @@ -951,7 +969,7 @@ def install( version_str = build_version_str(entry.evr) target_file = \ '%s%s-%s.rpm' % (PACKAGE_NAME_PREFIX, name, version_str) - unverified_rpm_list.append( + unverified_rpm_list[name] = ( (os.path.join(args.cachedir, target_file), entry.reponame)) dl_list = dl_list_copy @@ -969,15 +987,14 @@ def install( 'This will override changes made in the following VMs:', override_tpls) - with tempfile.TemporaryDirectory(dir=args.cachedir) as dl_dir: - download(args, app, path_override=dl_dir, - dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, - version_selector=version_selector) + package_hdrs = download(args, app, + dl_list=dl_list, + version_selector=version_selector) - # Verify downloaded templates - for rpmfile, reponame in unverified_rpm_list: - verify(rpmfile, reponame, dl_dir=dl_dir) - unverified_rpm_list = [] + # Verify downloaded templates + for name, (rpmfile, reponame) in unverified_rpm_list.items(): + verify(rpmfile, reponame, package_hdrs[name]) + del unverified_rpm_list # Unpack and install for rpmfile, reponame, name, package_hdr in verified_rpm_list: From e6360da22e347181a706ff5026f98b86f840dbb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 19 Feb 2021 01:12:26 +0100 Subject: [PATCH 102/119] qvm-template: default confirm to 'n' Capital 'N' in the prompt suggests it is the default - really make it the default. --- qubesadmin/tools/qvm_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 406f839..6eb65e4 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -368,7 +368,7 @@ def confirm_action(msg: str, affected: typing.List[str]) -> None: confirm = '' while confirm != 'y': confirm = input('Are you sure? [y/N] ').lower() - if confirm == 'n': + if confirm != 'y': print('command cancelled.') sys.exit(1) From bcf59579f16273e17a6c318d1db28b7e5c210525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 19 Feb 2021 14:02:53 +0100 Subject: [PATCH 103/119] qvm-template-postprocess: treat missing appmenus files as warnings only Do not fail if *-whitelisted-appmenus.list files are not included in the template package, only log an error. While at it, use pathlib there to make the code a bit nicer. --- qubesadmin/tools/qvm_template_postprocess.py | 39 +++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py index 3b1976a..d44f704 100644 --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -23,6 +23,7 @@ import asyncio import glob import os +import pathlib import shutil import subprocess @@ -166,16 +167,26 @@ def import_appmenus(vm, source_dir, skip_generate=True): # store the whitelists in VM features # separated by spaces should be ok as there should be no spaces in the file # name according to the FreeDesktop spec - with open(os.path.join(source_dir, 'vm-whitelisted-appmenus.list'), 'r') \ - as fd: - vm.features['default-menu-items'] = ' '.join([x.rstrip() for x in fd]) - with open(os.path.join(source_dir, 'whitelisted-appmenus.list'), 'r') \ - as fd: - vm.features['menu-items'] = ' '.join([x.rstrip() for x in fd]) - with open( - os.path.join(source_dir, 'netvm-whitelisted-appmenus.list'), 'r') \ - as fd: - vm.features['netvm-menu-items'] = ' '.join([x.rstrip() for x in fd]) + source_dir = pathlib.Path(source_dir) + try: + with open(source_dir / 'vm-whitelisted-appmenus.list', 'r') as fd: + vm.features['default-menu-items'] = \ + ' '.join([x.rstrip() for x in fd]) + except FileNotFoundError as e: + vm.log.warning('Cannot set default-menu-items, %s not found', + e.filename) + try: + with open(source_dir / 'whitelisted-appmenus.list', 'r') as fd: + vm.features['menu-items'] = ' '.join([x.rstrip() for x in fd]) + except FileNotFoundError as e: + vm.log.warning('Cannot set menu-items, %s not found', + e.filename) + try: + with open(source_dir / 'netvm-whitelisted-appmenus.list', 'r') as fd: + vm.features['netvm-menu-items'] = ' '.join([x.rstrip() for x in fd]) + except FileNotFoundError as e: + vm.log.warning('Cannot set netvm-menu-items, %s not found', + e.filename) if skip_generate: return @@ -184,11 +195,11 @@ def import_appmenus(vm, source_dir, skip_generate=True): # implemented try: subprocess.check_call(cmd_prefix + ['qvm-appmenus', - '--set-default-whitelist={}'.format(os.path.join(source_dir, - 'vm-whitelisted-appmenus.list')), vm.name]) + '--set-default-whitelist={!s}'.format( + source_dir / 'vm-whitelisted-appmenus.list'), vm.name]) subprocess.check_call(cmd_prefix + ['qvm-appmenus', - '--set-whitelist={}'.format(os.path.join(source_dir, - 'whitelisted-appmenus.list')), vm.name]) + '--set-whitelist={!s}'.format( + source_dir / 'whitelisted-appmenus.list'), vm.name]) except subprocess.CalledProcessError as e: vm.log.warning('Failed to set default application list: %s', e) From c4efdf41c54115e7d5d8480aac17fe2c9d9c2549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 19 Feb 2021 14:12:53 +0100 Subject: [PATCH 104/119] qvm-template-postprocess: extract config handling into separate function Keep post_install() short. --- qubesadmin/tools/qvm_template_postprocess.py | 97 +++++++++++--------- 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py index d44f704..dca649a 100644 --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -309,48 +309,7 @@ def post_install(args): conf_path = os.path.join(args.dir, 'template.conf') if os.path.exists(conf_path): - conf = parse_template_config(conf_path) - # Import qvm-feature tags - for key in ( - 'no-monitor-layout', - 'pci-e820-host', - 'linux-stubdom', - 'gui', - 'gui-emulated' - 'qrexec'): - if key in conf: - if conf[key] == '1': - vm.features[key] = conf[key] - else: - vm.log.warning( - 'ignoring boolean config flags that are not \'1\'') - for key in ( - 'net.fake-ip', - 'net.fake-gateway', - 'net.fake-netmask'): - if key in conf: - if validate_ip(conf[key]): - vm.features[key] = conf[key] - else: - vm.log.warning( - 'ignoring invalid value for \'%s\'', key) - if 'virt-mode' in conf: - if conf['virt-mode'] == 'pv' and args.allow_pv: - vm.virt_mode = 'pv' - elif conf['virt-mode'] == 'pv': - vm.log.warning( - '--allow-pv not set, ignoring request to change virt-mode') - elif conf['virt-mode'] in ('pvh', 'hvm'): - vm.virt_mode = conf['virt-mode'] - else: - vm.log.warning('ignoring invalid value for virt-mode') - - if 'kernel' in conf: - if conf['kernel'] == '': - vm.kernel = '' - else: - vm.log.warning( - 'Currently only supports setting kernel to (none)') + import_template_config(args, conf_path, vm) if not args.skip_start: yield from call_postinstall_service(vm) @@ -372,6 +331,60 @@ def post_install(args): return 0 +def import_template_config(args, conf_path, vm): + """ + Parse template.conf and apply its content to the just installed TemplateVM + + :param args: arguments for qvm-template-postprocess (used for --allow-pv + option and possibly some other in the future) + :param conf_path: path to the template.conf + :param vm: Template to operate on + :return: + """ + conf = parse_template_config(conf_path) + # Import qvm-feature tags + for key in ( + 'no-monitor-layout', + 'pci-e820-host', + 'linux-stubdom', + 'gui', + 'gui-emulated' + 'qrexec'): + if key in conf: + if conf[key] == '1': + vm.features[key] = conf[key] + else: + vm.log.warning( + 'ignoring boolean config flags that are not \'1\'') + for key in ( + 'net.fake-ip', + 'net.fake-gateway', + 'net.fake-netmask'): + if key in conf: + if validate_ip(conf[key]): + vm.features[key] = conf[key] + else: + vm.log.warning( + 'ignoring invalid value for \'%s\'', key) + if 'virt-mode' in conf: + if conf['virt-mode'] == 'pv' and args.allow_pv: + vm.virt_mode = 'pv' + elif conf['virt-mode'] == 'pv': + vm.log.warning( + '--allow-pv not set, ignoring request to change virt-mode') + elif conf['virt-mode'] in ('pvh', 'hvm'): + vm.virt_mode = conf['virt-mode'] + else: + vm.log.warning('ignoring invalid value for virt-mode') + + if 'kernel' in conf: + if conf['kernel'] == '': + vm.kernel = '' + else: + vm.log.warning( + 'Currently only supports setting kernel to (none)') + + def pre_remove(args): '''Handle pre-removal tasks''' app = args.app From 2c5572b3d9253c1ba4a0fcc6ff80420d888fa5f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 19 Feb 2021 14:58:30 +0100 Subject: [PATCH 105/119] qvm-template-postprocess: fix allowed features list Add missing coma, otherwise 'gui-emulated' and 'qrexec' were glued together. --- qubesadmin/tools/qvm_template_postprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template_postprocess.py b/qubesadmin/tools/qvm_template_postprocess.py index dca649a..b5b33d8 100644 --- a/qubesadmin/tools/qvm_template_postprocess.py +++ b/qubesadmin/tools/qvm_template_postprocess.py @@ -348,7 +348,7 @@ def import_template_config(args, conf_path, vm): 'pci-e820-host', 'linux-stubdom', 'gui', - 'gui-emulated' + 'gui-emulated', 'qrexec'): if key in conf: if conf[key] == '1': From b86408a36d9c57df65705d21ed646f446ad21fee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 19 Feb 2021 14:59:26 +0100 Subject: [PATCH 106/119] tests: qvm-template-postprocess - template.conf handling --- .../tests/tools/qvm_template_postprocess.py | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/qubesadmin/tests/tools/qvm_template_postprocess.py b/qubesadmin/tests/tools/qvm_template_postprocess.py index fded880..e20ef37 100644 --- a/qubesadmin/tests/tools/qvm_template_postprocess.py +++ b/qubesadmin/tests/tools/qvm_template_postprocess.py @@ -17,6 +17,7 @@ # # You should have received a copy of the GNU Lesser General Public License along # with this program; if not, see . +import argparse import asyncio import os import subprocess @@ -522,3 +523,119 @@ class TC_00_qvm_template_postprocess(qubesadmin.tests.QubesTestCase): 'post-install', 'test-vm', self.source_dir.name], app=self.app) self.assertAllCalled() + + def test_050_template_config(self): + template_config = """gui=1 +qrexec=1 +linux-stubdom=1 +net.fake-ip=192.168.1.100 +net.fake-netmask=255.255.255.0 +net.fake-gateway=192.168.1.1 +virt-mode=hvm +kernel= +""" + template_conf = os.path.join(self.source_dir.name, 'template.conf') + with open(template_conf, 'w') as f: + f.write(template_config) + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\0test-vm class=TemplateVM state=Halted\n' + self.app.expected_calls[( + 'test-vm', 'admin.vm.feature.Set', 'gui', b'1')] = b'0\0' + self.app.expected_calls[( + 'test-vm', 'admin.vm.feature.Set', 'qrexec', b'1')] = b'0\0' + self.app.expected_calls[( + 'test-vm', 'admin.vm.feature.Set', 'linux-stubdom', b'1')] = b'0\0' + self.app.expected_calls[( + 'test-vm', 'admin.vm.feature.Set', 'net.fake-ip', b'192.168.1.100')] = b'0\0' + self.app.expected_calls[( + 'test-vm', 'admin.vm.feature.Set', 'net.fake-netmask', b'255.255.255.0')] = b'0\0' + self.app.expected_calls[( + 'test-vm', 'admin.vm.feature.Set', 'net.fake-gateway', b'192.168.1.1')] = b'0\0' + self.app.expected_calls[( + 'test-vm', 'admin.vm.property.Set', 'virt_mode', b'hvm')] = b'0\0' + self.app.expected_calls[( + 'test-vm', 'admin.vm.property.Set', 'kernel', b'')] = b'0\0' + + vm = self.app.domains['test-vm'] + args = argparse.Namespace( + allow_pv=False, + ) + qubesadmin.tools.qvm_template_postprocess.import_template_config( + args, template_conf, vm) + self.assertAllCalled() + + def test_051_template_config_invalid(self): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\0test-vm class=TemplateVM state=Halted\n' + vm = self.app.domains['test-vm'] + args = argparse.Namespace( + allow_pv=False, + ) + with self.subTest('invalid feature value'): + template_config = "gui=false\n" + template_conf = os.path.join(self.source_dir.name, 'template.conf') + with open(template_conf, 'w') as f: + f.write(template_config) + qubesadmin.tools.qvm_template_postprocess.import_template_config( + args, template_conf, vm) + + with self.subTest('invalid feature name'): + template_config = "invalid=1\n" + template_conf = os.path.join(self.source_dir.name, 'template.conf') + with open(template_conf, 'w') as f: + f.write(template_config) + qubesadmin.tools.qvm_template_postprocess.import_template_config( + args, template_conf, vm) + + with self.subTest('invalid ip'): + template_config = "net.fake-ip=1.2.3.4.5\n" + template_conf = os.path.join(self.source_dir.name, 'template.conf') + with open(template_conf, 'w') as f: + f.write(template_config) + qubesadmin.tools.qvm_template_postprocess.import_template_config( + args, template_conf, vm) + + with self.subTest('invalid virt-mode'): + template_config = "virt-mode=invalid\n" + template_conf = os.path.join(self.source_dir.name, 'template.conf') + with open(template_conf, 'w') as f: + f.write(template_config) + qubesadmin.tools.qvm_template_postprocess.import_template_config( + args, template_conf, vm) + + with self.subTest('invalid kernel'): + template_config = "kernel=1.2.3.4.5\n" + template_conf = os.path.join(self.source_dir.name, 'template.conf') + with open(template_conf, 'w') as f: + f.write(template_config) + qubesadmin.tools.qvm_template_postprocess.import_template_config( + args, template_conf, vm) + self.assertAllCalled() + + def test_052_template_config_virt_mode_pv(self): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\0test-vm class=TemplateVM state=Halted\n' + vm = self.app.domains['test-vm'] + args = argparse.Namespace( + allow_pv=False, + ) + with self.subTest('not allowed'): + template_config = "virt-mode=pv\n" + template_conf = os.path.join(self.source_dir.name, 'template.conf') + with open(template_conf, 'w') as f: + f.write(template_config) + qubesadmin.tools.qvm_template_postprocess.import_template_config( + args, template_conf, vm) + with self.subTest('allowed'): + args = argparse.Namespace( + allow_pv=True, + ) + self.app.expected_calls[( + 'test-vm', 'admin.vm.property.Set', 'virt_mode', b'pv')] = b'0\0' + template_config = "virt-mode=pv\n" + template_conf = os.path.join(self.source_dir.name, 'template.conf') + with open(template_conf, 'w') as f: + f.write(template_config) + qubesadmin.tools.qvm_template_postprocess.import_template_config( + args, template_conf, vm) + self.assertAllCalled() From f4e826e65dc7de8a2b40fbae1201dbb4c08f8c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 19 Feb 2021 15:26:34 +0100 Subject: [PATCH 107/119] qvm-template: mute pylint complains about typing.NamedTuple This is false positive, PyCQA/pylint#3732 --- qubesadmin/tools/qvm_template.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 6eb65e4..9991d13 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -254,6 +254,8 @@ class VersionSelector(enum.Enum): LATEST_HIGHER = enum.auto() """Upgrade to the highest version that is higher than the current one.""" + +# pylint: disable=too-few-public-methods,inherit-non-class class Template(typing.NamedTuple): """Details of a template.""" name: str @@ -273,11 +275,14 @@ class Template(typing.NamedTuple): """Return a tuple of (EPOCH, VERSION, RELEASE)""" return self.epoch, self.version, self.release + class DlEntry(typing.NamedTuple): """Information about a template to be downloaded.""" evr: typing.Tuple[str, str, str] reponame: str dlsize: int +# pylint: enable=too-few-public-methods,inherit-non-class + def build_version_str(evr: typing.Tuple[str, str, str]) -> str: """Return version string described by ``evr``, which is in (epoch, version, From e00f35b9c31356bb40078bc0b27b7fed33d17c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 19 Feb 2021 23:06:07 +0100 Subject: [PATCH 108/119] tests: some more for qvm-template QubesOS/qubes-issues# --- qubesadmin/tests/tools/qvm_template.py | 201 +++++++++++++++++++++++-- 1 file changed, 192 insertions(+), 9 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 2746028..8fd263f 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -46,11 +46,12 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): hdr = { rpm.RPMTAG_SIGPGP: 'xxx', # non-empty rpm.RPMTAG_SIGGPG: 'xxx', # non-empty + rpm.RPMTAG_NAME: 'qubes-template-test-vm', } mock_ts.return_value.hdrFromFdno.return_value = hdr mock_proc.return_value = b'dummy.rpm: digests signatures OK\n' ret = qubesadmin.tools.qvm_template.verify_rpm('/dev/null', - ['/path/to/key']) + '/path/to/key', template_name='test-vm') mock_call.assert_called_once() mock_proc.assert_called_once() self.assertEqual(hdr, ret) @@ -69,7 +70,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_proc.return_value = b'dummy.rpm: digests OK\n' with self.assertRaises(Exception) as e: qubesadmin.tools.qvm_template.verify_rpm('/dev/null', - ['/path/to/key']) + '/path/to/key') mock_call.assert_called_once() mock_proc.assert_called_once() self.assertIn('Signature verification failed', e.exception.args[0]) @@ -88,7 +89,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_ts.return_value.hdrFromFdno.return_value = hdr mock_proc.return_value = b'dummy.rpm: digests OK\n' ret = qubesadmin.tools.qvm_template.verify_rpm('/dev/null', - ['/path/to/key'], True) + '/path/to/key', True) mock_proc.assert_not_called() mock_call.assert_not_called() self.assertEqual(ret, hdr) @@ -102,7 +103,28 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ['rpmkeys', '--checksig'], b'/dev/null: digests SIGNATURES NOT OK\n') with self.assertRaises(Exception) as e: qubesadmin.tools.qvm_template.verify_rpm('/dev/null', - ['/path/to/key']) + '/path/to/key') + mock_call.assert_called_once() + mock_proc.assert_called_once() + self.assertIn('Signature verification failed', e.exception.args[0]) + mock_ts.assert_not_called() + self.assertAllCalled() + + @mock.patch('rpm.TransactionSet') + @mock.patch('subprocess.check_call') + @mock.patch('subprocess.check_output') + def test_004_verify_rpm_badname(self, mock_proc, mock_call, mock_ts): + mock_proc.side_effect = subprocess.CalledProcessError(1, + ['rpmkeys', '--checksig'], b'/dev/null: digests signatures OK\n') + hdr = { + rpm.RPMTAG_SIGPGP: 'xxx', # non-empty + rpm.RPMTAG_SIGGPG: 'xxx', # non-empty + rpm.RPMTAG_NAME: 'qubes-template-unexpected', + } + mock_ts.return_value.hdrFromFdno.return_value = hdr + with self.assertRaises(Exception) as e: + qubesadmin.tools.qvm_template.verify_rpm('/dev/null', + '/path/to/key', template_name='test-vm') mock_call.assert_called_once() mock_proc.assert_called_once() self.assertIn('Signature verification failed', e.exception.args[0]) @@ -3677,7 +3699,7 @@ test-vm : Qubes template for fedora-31 @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') - def test_189_download_verify_fail(self, mock_qrexec, mock_dllist, + def test_190_download_fail_verify(self, mock_qrexec, mock_dllist, mock_verify_rpm): mock_qrexec.side_effect = self._mock_qrexec_download mock_verify_rpm.side_effect = \ @@ -3689,7 +3711,7 @@ test-vm : Qubes template for fedora-31 repo_files=[], keyring='/tmp/keyring.gpg', releasever='4.1', - nogpgcheck=False, + nogpgcheck=True, # make sure it gets ignored downloaddir=dir ) with self.assertRaises(qubesadmin.tools.qvm_template.SignatureVerificationError): @@ -3705,7 +3727,168 @@ test-vm : Qubes template for fedora-31 ]) self.assertEqual(mock_dllist.mock_calls, []) self.assertEqual(os.listdir(dir), []) + self.assertEqual(mock_verify_rpm.mock_calls, [ + mock.call(re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), + '/tmp/keyring.gpg', template_name='fedora-31'), + ]) - # TODO: test unexpected name downloaded - # TODO: test truncated download - # TODO: test no disk space \ No newline at end of file + def _mock_qrexec_download_short(self, args, app, spec, path, + dlsize=None, refresh=False): + self.assertFalse(os.path.exists(path), + '{} should not exist before'.format(path)) + # just create an empty file + with open(path, 'wb') as f: + if f is not None: + f.truncate(dlsize // 2) + + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.qrexec_download') + def test_191_download_fail_short(self, mock_qrexec, mock_dllist, + mock_verify_rpm): + mock_qrexec.side_effect = self._mock_qrexec_download_short + with tempfile.TemporaryDirectory() as dir, \ + self.assertRaises(SystemExit): + args = argparse.Namespace( + repo_files=[], + keyring='/tmp/keyring.gpg', + releasever='4.1', + nogpgcheck=False, + retries=1 + ) + qubesadmin.tools.qvm_template.download(args, self.app, dir, { + 'fedora-31': qubesadmin.tools.qvm_template.DlEntry( + ('1', '2', '3'), 'qubes-templates-itl', 1048576), + }) + self.assertEqual(mock_qrexec.mock_calls, [ + mock.call(args, self.app, 'qubes-template-fedora-31-1:2-3', + re_str(dir + '/.*/qubes-template-fedora-31-1:2-3.rpm.UNTRUSTED'), + 1048576), + ]) + self.assertEqual(mock_dllist.mock_calls, []) + self.assertEqual(mock_verify_rpm.mock_calls, []) + + + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + def test_200_reinstall_success( + self, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=TemplateVM state=Halted\n' + build_time = '2020-09-01 14:30:00' # 1598970600 + install_time = '2020-09-01 15:30:00' + for key, val in [ + ('name', 'test-vm'), + ('epoch', '2'), + ('version', '4.1'), + ('release', '2020')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + for key, val in [ + ('name', 'test-vm'), + ('epoch', '2'), + ('version', '4.1'), + ('release', '2020'), + ('reponame', 'qubes-templates-itl'), + ('buildtime', build_time), + ('installtime', install_time), + ('license', 'GPL'), + ('url', 'https://qubes-os.org'), + ('summary', 'Summary'), + ('description', 'Desc|desc')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Set', + f'template-{key}', + val.encode())] = b'0\0' + rpm_hdr = { + rpm.RPMTAG_NAME : 'qubes-template-test-vm', + rpm.RPMTAG_BUILDTIME : 1598970600, + rpm.RPMTAG_DESCRIPTION : 'Desc\ndesc', + rpm.RPMTAG_EPOCHNUM : 2, + rpm.RPMTAG_LICENSE : 'GPL', + rpm.RPMTAG_RELEASE : '2020', + rpm.RPMTAG_SUMMARY : 'Summary', + rpm.RPMTAG_URL : 'https://qubes-os.org', + rpm.RPMTAG_VERSION : '4.1' + } + mock_dl.return_value = {'test-vm': rpm_hdr} + dl_list = { + 'test-vm': qubesadmin.tools.qvm_template.DlEntry( + ('1', '4.1', '20200101'), 'qubes-templates-itl', 1048576) + } + mock_dl_list.return_value = dl_list + mock_call.side_effect = self.add_new_vm_side_effect + mock_time = mock.Mock(wraps=datetime.datetime) + mock_time.now.return_value = \ + datetime.datetime(2020, 9, 1, 15, 30, tzinfo=datetime.timezone.utc) + selector = qubesadmin.tools.qvm_template.VersionSelector.REINSTALL + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ + mock.patch('datetime.datetime', new=mock_time), \ + mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir: + args = argparse.Namespace( + templates=['test-vm'], + keyring='/tmp/keyring.gpg', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', + yes=False, + keep_cache=True, + allow_pv=False, + pool=None + ) + mock_tmpdir.return_value.__enter__.return_value = \ + '/var/tmp/qvm-template-tmpdir' + qubesadmin.tools.qvm_template.install(args, self.app, + version_selector=selector, + override_existing=True) + # Attempt to get download list + self.assertEqual(mock_dl_list.mock_calls, [ + mock.call(args, self.app, version_selector=selector) + ]) + mock_dl.assert_called_with(args, self.app, + dl_list=dl_list, version_selector=selector) + # already verified by download() + self.assertEqual(mock_verify.mock_calls, []) + # Package is extracted + mock_extract.assert_called_with('test-vm', + '/var/cache/qvm-template/qubes-template-test-vm-1:4.1-20200101.rpm', + '/var/tmp/qvm-template-tmpdir') + # Expect override confirmation + self.assertEqual(mock_confirm.mock_calls, + [mock.call(re_str(r'.*override changes.*:'), ['test-vm'])]) + # qvm-template-postprocess is called + self.assertEqual(mock_call.mock_calls, [ + mock.call([ + 'qvm-template-postprocess', + '--really', + '--no-installed-by-rpm', + 'post-install', + 'test-vm', + '/var/tmp/qvm-template-tmpdir' + '/var/lib/qubes/vm-templates/test-vm' + ]) + ]) + # Cache directory created + self.assertEqual(mock_mkdirs.mock_calls, [ + mock.call(args.cachedir, exist_ok=True) + ]) + self.assertAllCalled() \ No newline at end of file From ed3e368673b2d001bc0b59fd8c12f7f473647c37 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 20 Feb 2021 09:46:56 +0800 Subject: [PATCH 109/119] tests: add tests for qvm-template remove --- qubesadmin/tests/tools/qvm_template.py | 204 ++++++++++++++++++++++++- qubesadmin/tools/qvm_template.py | 5 +- 2 files changed, 207 insertions(+), 2 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 8fd263f..749e5f6 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -3891,4 +3891,206 @@ test-vm : Qubes template for fedora-31 self.assertEqual(mock_mkdirs.mock_calls, [ mock.call(args.cachedir, exist_ok=True) ]) - self.assertAllCalled() \ No newline at end of file + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_remove.main') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + def test_210_remove_success(self, mock_confirm, mock_remove): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = ( + b'0\x00vm1 class=TemplateVM state=Halted\n' + b'vm2 class=TemplateVM state=Halted\n' + ) + args = argparse.Namespace( + templates=['vm1', 'vm2'], + yes=False + ) + qubesadmin.tools.qvm_template.remove(args, self.app) + self.assertEqual(mock_confirm.mock_calls, + [mock.call(re_str(r'.*completely remove.*'), ['vm1', 'vm2'])]) + self.assertEqual(mock_remove.mock_calls, [ + mock.call(['--force', '--', 'vm1', 'vm2'], self.app) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_kill.main') + @mock.patch('qubesadmin.tools.qvm_remove.main') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + def test_211_remove_purge_disassoc_success( + self, + mock_confirm, + mock_remove, + mock_kill): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = ( + b'0\x00vm1 class=TemplateVM state=Halted\n' + b'vm2 class=TemplateVM state=Halted\n' + b'vm3 class=TemplateVM state=Halted\n' + b'vm4 class=TemplateVM state=Halted\n' + b'dummy class=TemplateVM state=Halted\n' + b'dummy-1 class=TemplateVM state=Halted\n' + ) + self.app.expected_calls[ + ('dummy', 'admin.vm.feature.Get', 'template-dummy', None)] = \ + b'0\x000' + self.app.expected_calls[ + ('dummy-1', 'admin.vm.feature.Get', + 'template-dummy', None)] = \ + b'0\x001' + self.app.expected_calls[ + ('vm2', 'admin.vm.property.Set', + 'default_template', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('vm2', 'admin.vm.property.Set', 'template', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('vm3', 'admin.vm.property.Set', 'netvm', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('vm3', 'admin.vm.property.Set', 'template', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('vm4', 'admin.vm.property.Set', 'netvm', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('vm4', 'admin.vm.property.Set', 'template', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('dom0', 'admin.property.Set', 'updatevm', b'')] = \ + b'0\x00' + args = argparse.Namespace( + templates=['vm1'], + yes=False + ) + def deps(app, vm): + if vm == 'vm1': + return [(self.app.domains['vm2'], 'default_template'), + (self.app.domains['vm3'], 'netvm')] + if vm == 'vm2' or vm == 'vm3': + return [(self.app.domains['vm4'], 'netvm')] + if vm == 'vm4': + return [(None, 'updatevm')] + return [] + with mock.patch('qubesadmin.utils.vm_dependencies') as mock_deps: + mock_deps.side_effect = deps + qubesadmin.tools.qvm_template.remove(args, self.app, purge=True) + # Once for purge (dependency detection) and + # one for disassoc (actually disassociating the dependencies + self.assertEqual(mock_deps.mock_calls, [ + mock.call(self.app, self.app.domains['vm1']), + mock.call(self.app, self.app.domains['vm2']), + mock.call(self.app, self.app.domains['vm3']), + mock.call(self.app, self.app.domains['vm4']), + mock.call(self.app, self.app.domains['vm1']), + mock.call(self.app, self.app.domains['vm2']), + mock.call(self.app, self.app.domains['vm3']), + mock.call(self.app, self.app.domains['vm4']) + ]) + self.assertEqual(mock_confirm.mock_calls, [ + mock.call(re_str(r'.*completely remove.*'), + ['vm1', 'vm2', 'vm3', 'vm4']), + mock.call(re_str(r'.*completely remove.*'), + ['vm1', 'vm2', 'vm3', 'vm4']), + mock.call(re_str(r'.*completely remove.*'), + ['vm1', 'vm2', 'vm3', 'vm4']) + ]) + self.assertEqual(mock_remove.mock_calls, [ + mock.call(['--force', '--', 'vm1', 'vm2', 'vm3', 'vm4', 'dummy-1'], + self.app) + ]) + self.assertEqual(mock_kill.mock_calls, [ + mock.call(['--', 'vm1', 'vm2', 'vm3', 'vm4', 'dummy-1'], self.app) + ]) + self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_kill.main') + @mock.patch('qubesadmin.tools.qvm_remove.main') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + def test_212_remove_disassoc_success( + self, + mock_confirm, + mock_remove, + mock_kill): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = ( + b'0\x00vm1 class=TemplateVM state=Halted\n' + b'vm2 class=TemplateVM state=Halted\n' + b'vm3 class=TemplateVM state=Halted\n' + b'vm4 class=TemplateVM state=Halted\n' + b'dummy class=TemplateVM state=Halted\n' + b'dummy-1 class=TemplateVM state=Halted\n' + ) + self.app.expected_calls[ + ('dummy', 'admin.vm.feature.Get', 'template-dummy', None)] = \ + b'0\x000' + self.app.expected_calls[ + ('dummy-1', 'admin.vm.feature.Get', + 'template-dummy', None)] = \ + b'0\x001' + self.app.expected_calls[ + ('vm2', 'admin.vm.property.Set', + 'default_template', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('vm2', 'admin.vm.property.Set', 'template', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('vm3', 'admin.vm.property.Set', 'netvm', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('vm3', 'admin.vm.property.Set', 'template', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('vm4', 'admin.vm.property.Set', 'netvm', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('vm4', 'admin.vm.property.Set', 'template', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('dom0', 'admin.property.Set', 'updatevm', b'')] = \ + b'0\x00' + args = argparse.Namespace( + templates=['vm1', 'vm2', 'vm3', 'vm4'], + yes=False + ) + def deps(app, vm): + if vm == 'vm1': + return [(self.app.domains['vm2'], 'default_template'), + (self.app.domains['vm3'], 'netvm')] + if vm == 'vm2' or vm == 'vm3': + return [(self.app.domains['vm4'], 'netvm')] + if vm == 'vm4': + return [(None, 'updatevm')] + return [] + with mock.patch('qubesadmin.utils.vm_dependencies') as mock_deps: + mock_deps.side_effect = deps + qubesadmin.tools.qvm_template.remove(args, self.app, disassoc=True) + self.assertEqual(mock_deps.mock_calls, [ + mock.call(self.app, self.app.domains['vm1']), + mock.call(self.app, self.app.domains['vm2']), + mock.call(self.app, self.app.domains['vm3']), + mock.call(self.app, self.app.domains['vm4']) + ]) + self.assertEqual(mock_confirm.mock_calls, [ + mock.call(re_str(r'.*completely remove.*'), + ['vm1', 'vm2', 'vm3', 'vm4']) + ]) + self.assertEqual(mock_remove.mock_calls, [ + mock.call(['--force', '--', 'vm1', 'vm2', 'vm3', 'vm4'], + self.app) + ]) + self.assertEqual(mock_kill.mock_calls, [ + mock.call(['--', 'vm1', 'vm2', 'vm3', 'vm4'], self.app) + ]) + self.assertAllCalled() + + def test_213_remove_fail_nodomain(self): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00vm1 class=TemplateVM state=Halted\n' + args = argparse.Namespace( + templates=['vm0'], + yes=False + ) + with mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + with self.assertRaises(SystemExit): + qubesadmin.tools.qvm_template.remove(args, self.app) + self.assertTrue('no such domain:' in mock_err.getvalue()) + self.assertAllCalled() diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 9991d13..9b1daa7 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -1360,13 +1360,16 @@ def remove( if not args.yes: repeat = 3 if purge else 1 + # XXX: Mutating the list later seems to break the tests... + remove_list_copy = remove_list.copy() for _ in range(repeat): confirm_action( 'This will completely remove the selected VM(s)...', - remove_list) + remove_list_copy) if disassoc: # Remove the dummy afterwards if we're purging + # as nothing should depend on it in the end remove_dummy = purge # Create dummy template; handle name collisions orig_dummy = dummy From 33d205c1dd4ded82e83b1439a6e5a43be3ba8604 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 20 Feb 2021 09:56:36 +0800 Subject: [PATCH 110/119] tests: fix tests for verify_rpm involving incorrect template names --- qubesadmin/tests/tools/qvm_template.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 749e5f6..6fa0269 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -114,21 +114,22 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): @mock.patch('subprocess.check_call') @mock.patch('subprocess.check_output') def test_004_verify_rpm_badname(self, mock_proc, mock_call, mock_ts): - mock_proc.side_effect = subprocess.CalledProcessError(1, - ['rpmkeys', '--checksig'], b'/dev/null: digests signatures OK\n') + mock_proc.return_value = b'/dev/null: digests signatures OK\n' hdr = { rpm.RPMTAG_SIGPGP: 'xxx', # non-empty rpm.RPMTAG_SIGGPG: 'xxx', # non-empty rpm.RPMTAG_NAME: 'qubes-template-unexpected', } mock_ts.return_value.hdrFromFdno.return_value = hdr - with self.assertRaises(Exception) as e: + with self.assertRaises( + qubesadmin.tools.qvm_template.SignatureVerificationError) as e: qubesadmin.tools.qvm_template.verify_rpm('/dev/null', '/path/to/key', template_name='test-vm') mock_call.assert_called_once() mock_proc.assert_called_once() - self.assertIn('Signature verification failed', e.exception.args[0]) - mock_ts.assert_not_called() + self.assertIn('package does not match expected template name', + e.exception.args[0]) + mock_ts.assert_called_once() self.assertAllCalled() @mock.patch('subprocess.Popen') From 60f5ba0e23828e8455c07c2746671a30a562a578 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 20 Feb 2021 09:57:59 +0800 Subject: [PATCH 111/119] qvm-template: test != 1 instead of == 0 for template-dummy feature --- qubesadmin/tools/qvm_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 9b1daa7..804bf02 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -1376,7 +1376,7 @@ def remove( cnt = 1 while dummy in app.domains \ and app.domains[dummy].features.get( - 'template-dummy', '0') == '0': + 'template-dummy', '0') != '1': dummy = '%s-%d' % (orig_dummy, cnt) cnt += 1 if dummy not in app.domains: From 764a56ade1785906819150076d50ad45c3d0f8be Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sat, 20 Feb 2021 11:03:39 +0800 Subject: [PATCH 112/119] tests: add more tests re. install, remove, and get_keys_for_repos --- qubesadmin/tests/tools/qvm_template.py | 268 ++++++++++++++++++++++++- 1 file changed, 267 insertions(+), 1 deletion(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 6fa0269..8fc08ae 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -757,7 +757,6 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): self.assertEqual(mock_dl_list.mock_calls, [ mock.call(args, self.app, version_selector=selector) ]) - # Nothing downloaded mock_dl.assert_called_with(args, self.app, dl_list=dl_list, version_selector=selector) # download already verify the package internally @@ -788,6 +787,171 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): self.assertEqual(mock_rename.mock_calls, []) self.assertAllCalled() + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + def test_108_install_download_fail_exists( + self, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=TemplateVM state=Halted\n' + mock_dl.return_value = {'test-vm': { + rpm.RPMTAG_NAME : 'qubes-template-test-vm', + rpm.RPMTAG_BUILDTIME : 1598970600, + rpm.RPMTAG_DESCRIPTION : 'Desc\ndesc', + rpm.RPMTAG_EPOCHNUM : 2, + rpm.RPMTAG_LICENSE : 'GPL', + rpm.RPMTAG_RELEASE : '2020', + rpm.RPMTAG_SUMMARY : 'Summary', + rpm.RPMTAG_URL : 'https://qubes-os.org', + rpm.RPMTAG_VERSION : '4.1' + }} + dl_list = { + 'test-vm': qubesadmin.tools.qvm_template.DlEntry( + ('1', '4.1', '20200101'), 'qubes-templates-itl', 1048576) + } + mock_dl_list.return_value = dl_list + mock_time = mock.Mock(wraps=datetime.datetime) + mock_time.now.return_value = \ + datetime.datetime(2020, 9, 1, 15, 30, tzinfo=datetime.timezone.utc) + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ + mock.patch('datetime.datetime', new=mock_time), \ + mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + args = argparse.Namespace( + templates='test-vm', + keyring='/tmp/keyring.gpg', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', + yes=False, + keep_cache=True, + allow_pv=False, + pool=None + ) + mock_tmpdir.return_value.__enter__.return_value = \ + '/var/tmp/qvm-template-tmpdir' + qubesadmin.tools.qvm_template.install(args, self.app) + self.assertIn('already installed, skipping', mock_err.getvalue()) + # Attempt to get download list + selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST + self.assertEqual(mock_dl_list.mock_calls, [ + mock.call(args, self.app, version_selector=selector) + ]) + # Nothing downloaded nor installed + mock_dl.assert_called_with(args, self.app, + dl_list={}, version_selector=selector) + mock_verify.assert_not_called() + mock_extract.assert_not_called() + mock_confirm.assert_not_called() + mock_call.assert_not_called() + # Cache directory created + self.assertEqual(mock_mkdirs.mock_calls, [ + mock.call(args.cachedir, exist_ok=True) + ]) + # No templates downloaded, thus no renames needed + self.assertEqual(mock_rename.mock_calls, []) + self.assertAllCalled() + + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + def test_109_install_fail_extract( + self, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = b'0\0' + mock_verify.return_value = { + rpm.RPMTAG_NAME : 'qubes-template-test-vm', + rpm.RPMTAG_BUILDTIME : 1598970600, + rpm.RPMTAG_DESCRIPTION : 'Desc\ndesc', + rpm.RPMTAG_EPOCHNUM : 2, + rpm.RPMTAG_LICENSE : 'GPL', + rpm.RPMTAG_RELEASE : '2020', + rpm.RPMTAG_SUMMARY : 'Summary', + rpm.RPMTAG_URL : 'https://qubes-os.org', + rpm.RPMTAG_VERSION : '4.1' + } + mock_dl_list.return_value = {} + mock_call.side_effect = self.add_new_vm_side_effect + mock_time = mock.Mock(wraps=datetime.datetime) + mock_time.now.return_value = \ + datetime.datetime(2020, 9, 1, 15, 30, tzinfo=datetime.timezone.utc) + # Extraction error + mock_extract.return_value = False + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ + mock.patch('datetime.datetime', new=mock_time), \ + mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err, \ + tempfile.NamedTemporaryFile(suffix='.rpm') as template_file: + path = template_file.name + args = argparse.Namespace( + templates=[path], + keyring='/tmp/keyring.gpg', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', + yes=False, + allow_pv=False, + pool=None + ) + mock_tmpdir.return_value.__enter__.return_value = \ + '/var/tmp/qvm-template-tmpdir' + with self.assertRaises(Exception) as e: + qubesadmin.tools.qvm_template.install(args, self.app) + self.assertIn('Failed to extract', e.exception.args[0]) + + # Attempt to get download list + selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST + self.assertEqual(mock_dl_list.mock_calls, [ + mock.call(args, self.app, version_selector=selector) + ]) + # Nothing downloaded + mock_dl.assert_called_with(args, self.app, + dl_list={}, version_selector=selector) + mock_verify.assert_called_once_with(template_file.name, + '/tmp/keyring.gpg', + nogpgcheck=False) + # Package is (attempted to be) extracted + mock_extract.assert_called_with('test-vm', path, + '/var/tmp/qvm-template-tmpdir') + # No packages overwritten, so no confirm needed + self.assertEqual(mock_confirm.mock_calls, []) + # No VM created + mock_call.assert_not_called() + # Cache directory created + self.assertEqual(mock_mkdirs.mock_calls, [ + mock.call(args.cachedir, exist_ok=True) + ]) + # No templates downloaded, thus no renames needed + self.assertEqual(mock_rename.mock_calls, []) + self.assertAllCalled() + def test_110_qrexec_payload_refresh_success(self): with tempfile.NamedTemporaryFile() as repo_conf1, \ tempfile.NamedTemporaryFile() as repo_conf2: @@ -4095,3 +4259,105 @@ test-vm : Qubes template for fedora-31 qubesadmin.tools.qvm_template.remove(args, self.app) self.assertTrue('no such domain:' in mock_err.getvalue()) self.assertAllCalled() + + @mock.patch('qubesadmin.tools.qvm_kill.main') + @mock.patch('qubesadmin.tools.qvm_remove.main') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + def test_214_remove_disassoc_success_newdummy( + self, + mock_confirm, + mock_remove, + mock_kill): + def append_new_vm_side_effect(*args, **kwargs): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] += \ + b'dummy-1 class=TemplateVM state=Halted\n' + self.app.domains.clear_cache() + return self.app.domains['dummy-1'] + self.app.add_new_vm = mock.Mock(side_effect=append_new_vm_side_effect) + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = ( + b'0\x00vm1 class=TemplateVM state=Halted\n' + b'vm2 class=TemplateVM state=Halted\n' + b'dummy class=TemplateVM state=Halted\n' + ) + self.app.expected_calls[ + ('dummy', 'admin.vm.feature.Get', 'template-dummy', None)] = \ + b'0\x000' + self.app.expected_calls[ + ('dummy-1', 'admin.vm.feature.Set', + 'template-dummy', b'1')] = \ + b'0\x00' + self.app.expected_calls[ + ('vm2', 'admin.vm.property.Set', + 'default_template', b'dummy-1')] = \ + b'0\x00' + self.app.expected_calls[ + ('vm2', 'admin.vm.property.Set', + 'template', b'dummy-1')] = \ + b'0\x00' + args = argparse.Namespace( + templates=['vm1'], + yes=False + ) + def deps(app, vm): + if vm == 'vm1': + return [(self.app.domains['vm2'], 'default_template')] + return [] + with mock.patch('qubesadmin.utils.vm_dependencies') as mock_deps: + mock_deps.side_effect = deps + qubesadmin.tools.qvm_template.remove(args, self.app, disassoc=True) + self.assertEqual(mock_deps.mock_calls, [ + mock.call(self.app, self.app.domains['vm1']) + ]) + self.assertEqual(mock_confirm.mock_calls, [ + mock.call(re_str(r'.*completely remove.*'), ['vm1']) + ]) + self.assertEqual(mock_remove.mock_calls, [ + mock.call(['--force', '--', 'vm1'], self.app) + ]) + self.assertEqual(mock_kill.mock_calls, [ + mock.call(['--', 'vm1'], self.app) + ]) + self.assertAllCalled() + + def test_220_get_keys_for_repos_success(self): + with tempfile.NamedTemporaryFile() as f: + f.write( +b'''[qubes-templates-itl] +name = Qubes Templates repository +#baseurl = https://yum.qubes-os.org/r$releasever/templates-itl +#baseurl = http://yum.qubesosfasa4zl44o4tws22di6kepyzfeqv3tg4e3ztknltfxqrymdad.onion/r$releasever/templates-itl +metalink = https://yum.qubes-os.org/r$releasever/templates-itl/repodata/repomd.xml.metalink +enabled = 1 +fastestmirror = 1 +metadata_expire = 7d +gpgcheck = 1 +gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +[qubes-templates-itl-testing-nokey] +name = Qubes Templates repository +#baseurl = https://yum.qubes-os.org/r$releasever/templates-itl-testing +#baseurl = http://yum.qubesosfasa4zl44o4tws22di6kepyzfeqv3tg4e3ztknltfxqrymdad.onion/r$releasever/templates-itl-testing +metalink = https://yum.qubes-os.org/r$releasever/templates-itl-testing/repodata/repomd.xml.metalink +enabled = 0 +fastestmirror = 1 +gpgcheck = 1 +#gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary +[qubes-templates-itl-testing] +name = Qubes Templates repository +#baseurl = https://yum.qubes-os.org/r$releasever/templates-itl-testing +#baseurl = http://yum.qubesosfasa4zl44o4tws22di6kepyzfeqv3tg4e3ztknltfxqrymdad.onion/r$releasever/templates-itl-testing +metalink = https://yum.qubes-os.org/r$releasever/templates-itl-testing/repodata/repomd.xml.metalink +enabled = 0 +fastestmirror = 1 +gpgcheck = 1 +gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-primary-testing +''') + f.flush() + ret = qubesadmin.tools.qvm_template.get_keys_for_repos( + [f.name], 'r4.1') + self.assertEqual(ret, { + 'qubes-templates-itl': + '/etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-r4.1-primary', + 'qubes-templates-itl-testing': + '/etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-r4.1-primary-testing' + }) + self.assertAllCalled() From dedf5ac6e6232cfa68b07e0139a019b265c8eb2c Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 21 Feb 2021 02:09:23 +0800 Subject: [PATCH 113/119] qvm-template: only ask for confirmation during install if something is being done --- qubesadmin/tools/qvm_template.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 804bf02..07dd330 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -988,9 +988,12 @@ def install( for name in dl_list: override_tpls.append(name) - confirm_action( - 'This will override changes made in the following VMs:', - override_tpls) + # Only confirm if we have something to do + # since confiming w/ an empty list is probably silly + if override_tpls: + confirm_action( + 'This will override changes made in the following VMs:', + override_tpls) package_hdrs = download(args, app, dl_list=dl_list, From fc520f8ed4440e62eb92b449500f7a7ada3ef75e Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 21 Feb 2021 02:11:23 +0800 Subject: [PATCH 114/119] qvm-template: update comments to reflect e424c7d --- qubesadmin/tools/qvm_template.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 07dd330..528712c 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -882,8 +882,7 @@ def install( unverified_rpm_list = [] # rpmfile, reponame verified_rpm_list = [] def verify(rpmfile, reponame, package_hdr=None): - """Verify package signature and version, remove "unverified" - suffix, and parse package header. + """Verify package signature and version and parse package header. If package_hdr is provided, the signature check is skipped and only other checks are performed.""" From a9d03d199b6bbc7627adcf85911f66e6350e2143 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 21 Feb 2021 02:13:16 +0800 Subject: [PATCH 115/119] tests: fix mock return values of get_dl_list when testing `qvm-template reinstall` --- qubesadmin/tests/tools/qvm_template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 8fc08ae..a295631 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -3997,7 +3997,7 @@ test-vm : Qubes template for fedora-31 mock_dl.return_value = {'test-vm': rpm_hdr} dl_list = { 'test-vm': qubesadmin.tools.qvm_template.DlEntry( - ('1', '4.1', '20200101'), 'qubes-templates-itl', 1048576) + ('2', '4.1', '2020'), 'qubes-templates-itl', 1048576) } mock_dl_list.return_value = dl_list mock_call.side_effect = self.add_new_vm_side_effect @@ -4035,7 +4035,7 @@ test-vm : Qubes template for fedora-31 self.assertEqual(mock_verify.mock_calls, []) # Package is extracted mock_extract.assert_called_with('test-vm', - '/var/cache/qvm-template/qubes-template-test-vm-1:4.1-20200101.rpm', + '/var/cache/qvm-template/qubes-template-test-vm-2:4.1-2020.rpm', '/var/tmp/qvm-template-tmpdir') # Expect override confirmation self.assertEqual(mock_confirm.mock_calls, From 4083b74284236399b4b63510ca921b1d9e0b1823 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Sun, 21 Feb 2021 02:19:43 +0800 Subject: [PATCH 116/119] tests: add tests for qvm-template reinstall/up/downgrade when nothing needs to be done --- qubesadmin/tests/tools/qvm_template.py | 259 +++++++++++++++++++++++++ 1 file changed, 259 insertions(+) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index a295631..4f4c4aa 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -4058,6 +4058,95 @@ test-vm : Qubes template for fedora-31 ]) self.assertAllCalled() + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + def test_201_reinstall_fail_noversion( + self, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=TemplateVM state=Halted\n' + for key, val in [ + ('name', 'test-vm'), + ('epoch', '2'), + ('version', '4.1'), + ('release', '2021')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + rpm_hdr = { + rpm.RPMTAG_NAME : 'qubes-template-test-vm', + rpm.RPMTAG_BUILDTIME : 1598970600, + rpm.RPMTAG_DESCRIPTION : 'Desc\ndesc', + rpm.RPMTAG_EPOCHNUM : 2, + rpm.RPMTAG_LICENSE : 'GPL', + rpm.RPMTAG_RELEASE : '2020', + rpm.RPMTAG_SUMMARY : 'Summary', + rpm.RPMTAG_URL : 'https://qubes-os.org', + rpm.RPMTAG_VERSION : '4.1' + } + mock_dl.return_value = {'test-vm': rpm_hdr} + dl_list = { + 'test-vm': qubesadmin.tools.qvm_template.DlEntry( + ('1', '4.1', '2020'), 'qubes-templates-itl', 1048576) + } + mock_dl_list.return_value = dl_list + selector = qubesadmin.tools.qvm_template.VersionSelector.REINSTALL + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ + self.assertRaises(SystemExit) as e, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + args = argparse.Namespace( + templates=['test-vm'], + keyring='/tmp/keyring.gpg', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', + yes=False, + keep_cache=True, + allow_pv=False, + pool=None + ) + qubesadmin.tools.qvm_template.install(args, self.app, + version_selector=selector, + override_existing=True) + self.assertIn( + 'Same version of template \'test-vm\' not found', + mock_err.getvalue()) + # Attempt to get download list + self.assertEqual(mock_dl_list.mock_calls, [ + mock.call(args, self.app, version_selector=selector) + ]) + mock_dl.assert_called_with(args, self.app, + dl_list=dl_list, version_selector=selector) + # already verified by download() + self.assertEqual(mock_verify.mock_calls, []) + # Expect override confirmation + self.assertEqual(mock_confirm.mock_calls, + [mock.call(re_str(r'.*override changes.*:'), ['test-vm'])]) + # Nothing extracted / installed + mock_extract.assert_not_called() + mock_call.assert_not_called() + # Cache directory created + self.assertEqual(mock_mkdirs.mock_calls, [ + mock.call(args.cachedir, exist_ok=True) + ]) + self.assertAllCalled() + @mock.patch('qubesadmin.tools.qvm_remove.main') @mock.patch('qubesadmin.tools.qvm_template.confirm_action') def test_210_remove_success(self, mock_confirm, mock_remove): @@ -4361,3 +4450,173 @@ gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-pri '/etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-r4.1-primary-testing' }) self.assertAllCalled() + + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + def test_220_downgrade_skip_lower( + self, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=TemplateVM state=Halted\n' + for key, val in [ + ('name', 'test-vm'), + ('epoch', '2'), + ('version', '4.1'), + ('release', '2020')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + rpm_hdr = { + rpm.RPMTAG_NAME : 'qubes-template-test-vm', + rpm.RPMTAG_BUILDTIME : 1598970600, + rpm.RPMTAG_DESCRIPTION : 'Desc\ndesc', + rpm.RPMTAG_EPOCHNUM : 2, + rpm.RPMTAG_LICENSE : 'GPL', + rpm.RPMTAG_RELEASE : '2021', + rpm.RPMTAG_SUMMARY : 'Summary', + rpm.RPMTAG_URL : 'https://qubes-os.org', + rpm.RPMTAG_VERSION : '4.1' + } + mock_verify.return_value = rpm_hdr + mock_dl_list.return_value = {} + selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST_LOWER + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ + tempfile.NamedTemporaryFile(suffix='.rpm') as template_file, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + args = argparse.Namespace( + templates=[template_file.name], + keyring='/tmp/keyring.gpg', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', + yes=False, + keep_cache=True, + allow_pv=False, + pool=None + ) + qubesadmin.tools.qvm_template.install(args, self.app, + version_selector=selector, + override_existing=True) + self.assertIn( + 'lower version already installed', + mock_err.getvalue()) + # Attempt to get download list + self.assertEqual(mock_dl_list.mock_calls, [ + mock.call(args, self.app, version_selector=selector) + ]) + mock_dl.assert_called_with(args, self.app, + dl_list={}, version_selector=selector) + self.assertEqual(mock_verify.mock_calls, [ + mock.call(template_file.name, '/tmp/keyring.gpg', nogpgcheck=False) + ]) + # No confirmation since nothing needs to be done + mock_confirm.assert_not_called() + # Nothing extracted / installed + mock_extract.assert_not_called() + mock_call.assert_not_called() + # Cache directory created + self.assertEqual(mock_mkdirs.mock_calls, [ + mock.call(args.cachedir, exist_ok=True) + ]) + self.assertAllCalled() + + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + def test_221_upgrade_skip_higher( + self, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=TemplateVM state=Halted\n' + for key, val in [ + ('name', 'test-vm'), + ('epoch', '2'), + ('version', '4.1'), + ('release', '2021')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + rpm_hdr = { + rpm.RPMTAG_NAME : 'qubes-template-test-vm', + rpm.RPMTAG_BUILDTIME : 1598970600, + rpm.RPMTAG_DESCRIPTION : 'Desc\ndesc', + rpm.RPMTAG_EPOCHNUM : 2, + rpm.RPMTAG_LICENSE : 'GPL', + rpm.RPMTAG_RELEASE : '2020', + rpm.RPMTAG_SUMMARY : 'Summary', + rpm.RPMTAG_URL : 'https://qubes-os.org', + rpm.RPMTAG_VERSION : '4.1' + } + mock_verify.return_value = rpm_hdr + mock_dl_list.return_value = {} + selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST_HIGHER + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ + tempfile.NamedTemporaryFile(suffix='.rpm') as template_file, \ + mock.patch('sys.stderr', new=io.StringIO()) as mock_err: + args = argparse.Namespace( + templates=[template_file.name], + keyring='/tmp/keyring.gpg', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', + yes=False, + keep_cache=True, + allow_pv=False, + pool=None + ) + qubesadmin.tools.qvm_template.install(args, self.app, + version_selector=selector, + override_existing=True) + self.assertIn( + 'higher version already installed', + mock_err.getvalue()) + # Attempt to get download list + self.assertEqual(mock_dl_list.mock_calls, [ + mock.call(args, self.app, version_selector=selector) + ]) + mock_dl.assert_called_with(args, self.app, + dl_list={}, version_selector=selector) + self.assertEqual(mock_verify.mock_calls, [ + mock.call(template_file.name, '/tmp/keyring.gpg', nogpgcheck=False) + ]) + # No confirmation since nothing needs to be done + mock_confirm.assert_not_called() + # Nothing extracted / installed + mock_extract.assert_not_called() + mock_call.assert_not_called() + # Cache directory created + self.assertEqual(mock_mkdirs.mock_calls, [ + mock.call(args.cachedir, exist_ok=True) + ]) + self.assertAllCalled() From 64e9c240542062d350acc05d22e35bc184f5afce Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 2 Mar 2021 15:01:54 +0800 Subject: [PATCH 117/119] tests: improve TestProcess behavior - Have it actually write to the given stdout handle. - Return the return code for `poll` instead of returning `None`, so that the process is observed to terminate. --- qubesadmin/tests/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/qubesadmin/tests/__init__.py b/qubesadmin/tests/__init__.py index a0fa1e6..996276f 100644 --- a/qubesadmin/tests/__init__.py +++ b/qubesadmin/tests/__init__.py @@ -52,7 +52,7 @@ class TestVMCollection(dict): class TestProcess(object): - def __init__(self, input_callback=None, stdout=None, stderr=None): + def __init__(self, input_callback=None, stdout=None, stderr=None, stdout_data=None): self.input_callback = input_callback self.got_any_input = False self.stdin = io.BytesIO() @@ -70,6 +70,10 @@ class TestProcess(object): self.stderr = io.BytesIO() else: self.stderr = stderr + if stdout_data: + self.stdout.write(stdout_data) + # Seek to head so that it can be read later + self.stdout.seek(0) self.returncode = 0 def store_input(self): @@ -91,7 +95,7 @@ class TestProcess(object): return 0 def poll(self): - return None + return self.returncode class _AssertNotRaisesContext(object): @@ -167,11 +171,12 @@ class QubesTest(qubesadmin.app.QubesBase): # raise AssertionError('Unexpected service call {!r}'.format(call_key)) if call_key in self.expected_service_calls: kwargs = kwargs.copy() - kwargs['stdout'] = io.BytesIO(self.expected_service_calls[call_key]) + kwargs['stdout_data'] = self.expected_service_calls[call_key] return TestProcess(lambda input: self.service_calls.append((dest, service, input)), stdout=kwargs.get('stdout', None), stderr=kwargs.get('stderr', None), + stdout_data=kwargs.get('stdout_data', None), ) From d1ce8d3a95d13942205b3f910323808e74b8aca3 Mon Sep 17 00:00:00 2001 From: WillyPillow Date: Tue, 2 Mar 2021 15:14:08 +0800 Subject: [PATCH 118/119] tests: add tests for other qvm-template functions --- qubesadmin/tests/tools/qvm_template.py | 662 ++++++++++++++++++++++++- 1 file changed, 659 insertions(+), 3 deletions(-) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index 4f4c4aa..d6c2709 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -291,6 +291,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_tmpdir.return_value.__enter__.return_value = \ '/var/tmp/qvm-template-tmpdir' qubesadmin.tools.qvm_template.install(args, self.app) + # Downloaded package should not be removed + self.assertTrue(os.path.exists(path)) # Attempt to get download list selector = qubesadmin.tools.qvm_template.VersionSelector.LATEST self.assertEqual(mock_dl_list.mock_calls, [ @@ -675,6 +677,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): self.assertEqual(mock_rename.mock_calls, []) self.assertAllCalled() + @mock.patch('os.remove') @mock.patch('os.rename') @mock.patch('os.makedirs') @mock.patch('subprocess.check_call') @@ -692,7 +695,8 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock_confirm, mock_call, mock_mkdirs, - mock_rename): + mock_rename, + mock_remove): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = b'0\0' build_time = '2020-09-01 14:30:00' # 1598970600 install_time = '2020-09-01 15:30:00' @@ -745,7 +749,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): repo_files=[], releasever='4.1', yes=False, - keep_cache=True, + keep_cache=False, allow_pv=False, pool=None ) @@ -785,6 +789,12 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # No templates downloaded, thus no renames needed self.assertEqual(mock_rename.mock_calls, []) + # Downloaded template is removed + self.assertEqual(mock_remove.mock_calls, [ + mock.call('/var/cache/qvm-template/' \ + 'qubes-template-test-vm-1:4.1-20200101.rpm'), + mock.call('/tmp/test.lock') + ]) self.assertAllCalled() @mock.patch('os.rename') @@ -3934,6 +3944,7 @@ test-vm : Qubes template for fedora-31 self.assertEqual(mock_verify_rpm.mock_calls, []) + @mock.patch('os.remove') @mock.patch('os.rename') @mock.patch('os.makedirs') @mock.patch('subprocess.check_call') @@ -3951,7 +3962,8 @@ test-vm : Qubes template for fedora-31 mock_confirm, mock_call, mock_mkdirs, - mock_rename): + mock_rename, + mock_remove): self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ b'0\x00test-vm class=TemplateVM state=Halted\n' build_time = '2020-09-01 14:30:00' # 1598970600 @@ -4056,6 +4068,10 @@ test-vm : Qubes template for fedora-31 self.assertEqual(mock_mkdirs.mock_calls, [ mock.call(args.cachedir, exist_ok=True) ]) + # Downloaded package should not be removed + self.assertEqual(mock_remove.mock_calls, [ + mock.call('/tmp/test.lock') + ]) self.assertAllCalled() @mock.patch('os.rename') @@ -4147,6 +4163,131 @@ test-vm : Qubes template for fedora-31 ]) self.assertAllCalled() + @mock.patch('os.rename') + @mock.patch('os.makedirs') + @mock.patch('subprocess.check_call') + @mock.patch('qubesadmin.tools.qvm_template.confirm_action') + @mock.patch('qubesadmin.tools.qvm_template.extract_rpm') + @mock.patch('qubesadmin.tools.qvm_template.download') + @mock.patch('qubesadmin.tools.qvm_template.get_dl_list') + @mock.patch('qubesadmin.tools.qvm_template.verify_rpm') + def test_202_reinstall_local_success( + self, + mock_verify, + mock_dl_list, + mock_dl, + mock_extract, + mock_confirm, + mock_call, + mock_mkdirs, + mock_rename): + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=TemplateVM state=Halted\n' + build_time = '2020-09-01 14:30:00' # 1598970600 + install_time = '2020-09-01 15:30:00' + for key, val in [ + ('name', 'test-vm'), + ('epoch', '2'), + ('version', '4.1'), + ('release', '2020')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + for key, val in [ + ('name', 'test-vm'), + ('epoch', '2'), + ('version', '4.1'), + ('release', '2020'), + ('reponame', '@commandline'), + ('buildtime', build_time), + ('installtime', install_time), + ('license', 'GPL'), + ('url', 'https://qubes-os.org'), + ('summary', 'Summary'), + ('description', 'Desc|desc')]: + self.app.expected_calls[( + 'test-vm', + 'admin.vm.feature.Set', + f'template-{key}', + val.encode())] = b'0\0' + rpm_hdr = { + rpm.RPMTAG_NAME : 'qubes-template-test-vm', + rpm.RPMTAG_BUILDTIME : 1598970600, + rpm.RPMTAG_DESCRIPTION : 'Desc\ndesc', + rpm.RPMTAG_EPOCHNUM : 2, + rpm.RPMTAG_LICENSE : 'GPL', + rpm.RPMTAG_RELEASE : '2020', + rpm.RPMTAG_SUMMARY : 'Summary', + rpm.RPMTAG_URL : 'https://qubes-os.org', + rpm.RPMTAG_VERSION : '4.1' + } + mock_verify.return_value = rpm_hdr + dl_list = {} + mock_dl_list.return_value = dl_list + mock_call.side_effect = self.add_new_vm_side_effect + mock_time = mock.Mock(wraps=datetime.datetime) + mock_time.now.return_value = \ + datetime.datetime(2020, 9, 1, 15, 30, tzinfo=datetime.timezone.utc) + selector = qubesadmin.tools.qvm_template.VersionSelector.REINSTALL + with mock.patch('qubesadmin.tools.qvm_template.LOCK_FILE', '/tmp/test.lock'), \ + mock.patch('datetime.datetime', new=mock_time), \ + mock.patch('tempfile.TemporaryDirectory') as mock_tmpdir, \ + tempfile.NamedTemporaryFile(suffix='.rpm') as template_file: + path = template_file.name + args = argparse.Namespace( + templates=[path], + keyring='/tmp/keyring.gpg', + nogpgcheck=False, + cachedir='/var/cache/qvm-template', + repo_files=[], + releasever='4.1', + yes=False, + allow_pv=False, + pool=None + ) + mock_tmpdir.return_value.__enter__.return_value = \ + '/var/tmp/qvm-template-tmpdir' + qubesadmin.tools.qvm_template.install(args, self.app, + version_selector=selector, + override_existing=True) + # Package is extracted + mock_extract.assert_called_with( + 'test-vm', + path, + '/var/tmp/qvm-template-tmpdir') + # Package verified + self.assertEqual(mock_verify.mock_calls, [ + mock.call(path, '/tmp/keyring.gpg', nogpgcheck=False) + ]) + # Attempt to get download list + self.assertEqual(mock_dl_list.mock_calls, [ + mock.call(args, self.app, version_selector=selector) + ]) + mock_dl.assert_called_with(args, self.app, + dl_list=dl_list, version_selector=selector) + # Expect override confirmation + self.assertEqual(mock_confirm.mock_calls, + [mock.call(re_str(r'.*override changes.*:'), ['test-vm'])]) + # qvm-template-postprocess is called + self.assertEqual(mock_call.mock_calls, [ + mock.call([ + 'qvm-template-postprocess', + '--really', + '--no-installed-by-rpm', + 'post-install', + 'test-vm', + '/var/tmp/qvm-template-tmpdir' + '/var/lib/qubes/vm-templates/test-vm' + ]) + ]) + # Cache directory created + self.assertEqual(mock_mkdirs.mock_calls, [ + mock.call(args.cachedir, exist_ok=True) + ]) + self.assertAllCalled() + @mock.patch('qubesadmin.tools.qvm_remove.main') @mock.patch('qubesadmin.tools.qvm_template.confirm_action') def test_210_remove_success(self, mock_confirm, mock_remove): @@ -4620,3 +4761,518 @@ gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-pri mock.call(args.cachedir, exist_ok=True) ]) self.assertAllCalled() + + def test_230_filter_version_latest(self): + query_res = [ + qubesadmin.tools.qvm_template.Template( + 'fedora-31', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200102', + 'qubes-templates-itl-testing', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200102', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ) + ] + results = qubesadmin.tools.qvm_template.filter_version( + query_res, + self.app + ) + self.assertEqual(sorted(results), [ + qubesadmin.tools.qvm_template.Template( + 'fedora-31', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200102', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ) + ]) + self.assertAllCalled() + + def test_231_filter_version_reinstall(self): + query_res = [ + qubesadmin.tools.qvm_template.Template( + 'fedora-31', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200102', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200102', + 'qubes-templates-itl-testing', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ) + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00fedora-31 class=TemplateVM state=Halted\n' \ + b'fedora-32 class=TemplateVM state=Halted\n' + for key, val in [ + ('name', 'fedora-31'), + ('epoch', '0'), + ('version', '4.1'), + ('release', '20200101')]: + self.app.expected_calls[( + 'fedora-31', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + for key, val in [ + ('name', 'fedora-32'), + ('epoch', '0'), + ('version', '4.1'), + ('release', '20200101')]: + self.app.expected_calls[( + 'fedora-32', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + results = qubesadmin.tools.qvm_template.filter_version( + query_res, + self.app, + qubesadmin.tools.qvm_template.VersionSelector.REINSTALL + ) + self.assertEqual(sorted(results), [ + qubesadmin.tools.qvm_template.Template( + 'fedora-31', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ) + ]) + self.assertAllCalled() + + def test_232_filter_version_upgrade(self): + query_res = [ + qubesadmin.tools.qvm_template.Template( + 'fedora-31', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200102', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200102', + 'qubes-templates-itl-testing', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ) + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00fedora-31 class=TemplateVM state=Halted\n' \ + b'fedora-32 class=TemplateVM state=Halted\n' + for key, val in [ + ('name', 'fedora-31'), + ('epoch', '0'), + ('version', '4.1'), + ('release', '20200101')]: + self.app.expected_calls[( + 'fedora-31', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + for key, val in [ + ('name', 'fedora-32'), + ('epoch', '0'), + ('version', '4.1'), + ('release', '20200101')]: + self.app.expected_calls[( + 'fedora-32', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + results = qubesadmin.tools.qvm_template.filter_version( + query_res, + self.app, + qubesadmin.tools.qvm_template.VersionSelector.LATEST_HIGHER + ) + self.assertEqual(sorted(results), [ + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200102', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ) + ]) + self.assertAllCalled() + + def test_233_filter_version_downgrade(self): + query_res = [ + qubesadmin.tools.qvm_template.Template( + 'fedora-31', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-31', + 'Qubes template\n for fedora-31\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200102', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ), + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200102', + 'qubes-templates-itl-testing', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ) + ] + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00fedora-31 class=TemplateVM state=Halted\n' \ + b'fedora-32 class=TemplateVM state=Halted\n' + for key, val in [ + ('name', 'fedora-31'), + ('epoch', '0'), + ('version', '4.1'), + ('release', '20200101')]: + self.app.expected_calls[( + 'fedora-31', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + for key, val in [ + ('name', 'fedora-32'), + ('epoch', '0'), + ('version', '4.1'), + ('release', '20200102')]: + self.app.expected_calls[( + 'fedora-32', + 'admin.vm.feature.Get', + f'template-{key}', + None)] = b'0\0' + val.encode() + results = qubesadmin.tools.qvm_template.filter_version( + query_res, + self.app, + qubesadmin.tools.qvm_template.VersionSelector.LATEST_LOWER + ) + self.assertEqual(sorted(results), [ + qubesadmin.tools.qvm_template.Template( + 'fedora-32', + '0', + '4.1', + '20200101', + 'qubes-templates-itl', + 1048576, + datetime.datetime(2020, 1, 23, 4, 56), + 'GPL', + 'https://qubes-os.org', + 'Qubes template for fedora-32', + 'Qubes template\n for fedora-32\n' + ) + ]) + self.assertAllCalled() + + @mock.patch('os.path.exists') + def test_240_qubes_release(self, mock_exists): + # /usr/share/qubes/marker-vm does not exist + mock_exists.return_value = False + marker_vm = ''' +NAME=Qubes +VERSION="4.2 (R4.2)" +ID=qubes +# Some comments here +VERSION_ID=4.2 +PRETTY_NAME="Qubes 4.2 (R4.2)" +ANSI_COLOR="0;31" +CPE_NAME="cpe:/o:ITL:qubes:4.2" +''' + with mock.patch('builtins.open', mock.mock_open(read_data=marker_vm)) \ + as mock_open: + ret = qubesadmin.tools.qvm_template.qubes_release() + self.assertEqual(ret, '4.2') + self.assertEqual(mock_exists.mock_calls, [ + mock.call('/usr/share/qubes/marker-vm') + ]) + mock_open.assert_called_with('/etc/os-release', 'r') + self.assertAllCalled() + + @mock.patch('os.path.exists') + def test_241_qubes_release_quotes(self, mock_exists): + # /usr/share/qubes/marker-vm does not exist + mock_exists.return_value = False + os_rel = ''' +NAME=Qubes +VERSION="4.2 (R4.2)" +ID=qubes +# Some comments here +VERSION_ID="4.2" +PRETTY_NAME="Qubes 4.2 (R4.2)" +ANSI_COLOR="0;31" +CPE_NAME="cpe:/o:ITL:qubes:4.2" +''' + with mock.patch('builtins.open', mock.mock_open(read_data=os_rel)) \ + as mock_open: + ret = qubesadmin.tools.qvm_template.qubes_release() + self.assertEqual(ret, '4.2') + self.assertEqual(mock_exists.mock_calls, [ + mock.call('/usr/share/qubes/marker-vm') + ]) + mock_open.assert_called_with('/etc/os-release', 'r') + self.assertAllCalled() + + @mock.patch('os.path.exists') + def test_242_qubes_release_quotes(self, mock_exists): + # /usr/share/qubes/marker-vm does exist + mock_exists.return_value = True + marker_vm = ''' +# This is just a marker file for Qubes OS VM. +# This VM have tools for Qubes version: +4.2 +''' + with mock.patch('builtins.open', mock.mock_open(read_data=marker_vm)) \ + as mock_open: + ret = qubesadmin.tools.qvm_template.qubes_release() + self.assertEqual(ret, '4.2') + self.assertEqual(mock_exists.mock_calls, [ + mock.call('/usr/share/qubes/marker-vm') + ]) + mock_open.assert_called_with('/usr/share/qubes/marker-vm', 'r') + self.assertAllCalled() + + def test_250_qrexec_download_success(self): + rand_bytes = os.urandom(128) + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=TemplateVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateDownload')] = rand_bytes + args = argparse.Namespace( + repo_files=[], + releasever='4.1', + updatevm='test-vm', + enablerepo=[], + disablerepo=[], + repoid=[], + quiet=True + ) + with tempfile.NamedTemporaryFile() as fd: + qubesadmin.tools.qvm_template.qrexec_download( + args, self.app, 'fedora-31:4.0', path=fd.name) + with open(fd.name, 'rb') as fd2: + result = fd2.read() + self.assertEqual(rand_bytes, result) + self.assertAllCalled() + + def test_251_qrexec_download_fail(self): + rand_bytes = os.urandom(128) + self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \ + b'0\x00test-vm class=TemplateVM state=Halted\n' + self.app.expected_service_calls[ + ('test-vm', 'qubes.TemplateDownload')] = rand_bytes + args = argparse.Namespace( + repo_files=[], + releasever='4.1', + updatevm='test-vm', + enablerepo=[], + disablerepo=[], + repoid=[], + quiet=True + ) + with tempfile.NamedTemporaryFile() as fd, \ + mock.patch('qubesadmin.tests.TestProcess.wait') as mock_wait: + mock_wait.return_value = 1 + with self.assertRaises(ConnectionError): + qubesadmin.tools.qvm_template.qrexec_download( + args, self.app, 'fedora-31:4.0', path=fd.name) + self.assertAllCalled() From 9020f2e1fd0f26e2c19ada0beb9f33219e0a0283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 1 Apr 2021 01:31:35 +0200 Subject: [PATCH 119/119] qvm-template: fix downloading template for install Donwload a template into a cache dir, not into default of `qvm-template download` (current directory). --- qubesadmin/tests/tools/qvm_template.py | 11 +++++++++++ qubesadmin/tools/qvm_template.py | 1 + 2 files changed, 12 insertions(+) diff --git a/qubesadmin/tests/tools/qvm_template.py b/qubesadmin/tests/tools/qvm_template.py index d6c2709..e97d03c 100644 --- a/qubesadmin/tests/tools/qvm_template.py +++ b/qubesadmin/tests/tools/qvm_template.py @@ -300,6 +300,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # Nothing downloaded mock_dl.assert_called_with(args, self.app, + path_override='/var/cache/qvm-template', dl_list={}, version_selector=selector) mock_verify.assert_called_once_with(template_file.name, '/tmp/keyring.gpg', nogpgcheck=False) @@ -409,6 +410,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # Nothing downloaded mock_dl.assert_called_with(args, self.app, + path_override='/var/cache/qvm-template', dl_list={}, version_selector=selector) # Package is extracted mock_extract.assert_called_with('test-vm', path, @@ -554,6 +556,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): # Nothing downloaded self.assertEqual(mock_dl.mock_calls, [ mock.call(args, self.app, + path_override='/var/cache/qvm-template', dl_list={}, version_selector=selector) ]) # Should not be executed: @@ -762,6 +765,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): mock.call(args, self.app, version_selector=selector) ]) mock_dl.assert_called_with(args, self.app, + path_override='/var/cache/qvm-template', dl_list=dl_list, version_selector=selector) # download already verify the package internally self.assertEqual(mock_verify.mock_calls, []) @@ -863,6 +867,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # Nothing downloaded nor installed mock_dl.assert_called_with(args, self.app, + path_override='/var/cache/qvm-template', dl_list={}, version_selector=selector) mock_verify.assert_not_called() mock_extract.assert_not_called() @@ -943,6 +948,7 @@ class TC_00_qvm_template(qubesadmin.tests.QubesTestCase): ]) # Nothing downloaded mock_dl.assert_called_with(args, self.app, + path_override='/var/cache/qvm-template', dl_list={}, version_selector=selector) mock_verify.assert_called_once_with(template_file.name, '/tmp/keyring.gpg', @@ -4042,6 +4048,7 @@ test-vm : Qubes template for fedora-31 mock.call(args, self.app, version_selector=selector) ]) mock_dl.assert_called_with(args, self.app, + path_override='/var/cache/qvm-template', dl_list=dl_list, version_selector=selector) # already verified by download() self.assertEqual(mock_verify.mock_calls, []) @@ -4148,6 +4155,7 @@ test-vm : Qubes template for fedora-31 mock.call(args, self.app, version_selector=selector) ]) mock_dl.assert_called_with(args, self.app, + path_override='/var/cache/qvm-template', dl_list=dl_list, version_selector=selector) # already verified by download() self.assertEqual(mock_verify.mock_calls, []) @@ -4266,6 +4274,7 @@ test-vm : Qubes template for fedora-31 mock.call(args, self.app, version_selector=selector) ]) mock_dl.assert_called_with(args, self.app, + path_override='/var/cache/qvm-template', dl_list=dl_list, version_selector=selector) # Expect override confirmation self.assertEqual(mock_confirm.mock_calls, @@ -4662,6 +4671,7 @@ gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-pri mock.call(args, self.app, version_selector=selector) ]) mock_dl.assert_called_with(args, self.app, + path_override='/var/cache/qvm-template', dl_list={}, version_selector=selector) self.assertEqual(mock_verify.mock_calls, [ mock.call(template_file.name, '/tmp/keyring.gpg', nogpgcheck=False) @@ -4747,6 +4757,7 @@ gpgkey = file:///etc/qubes/repo-templates/keys/RPM-GPG-KEY-qubes-$releasever-pri mock.call(args, self.app, version_selector=selector) ]) mock_dl.assert_called_with(args, self.app, + path_override='/var/cache/qvm-template', dl_list={}, version_selector=selector) self.assertEqual(mock_verify.mock_calls, [ mock.call(template_file.name, '/tmp/keyring.gpg', nogpgcheck=False) diff --git a/qubesadmin/tools/qvm_template.py b/qubesadmin/tools/qvm_template.py index 528712c..e01770f 100644 --- a/qubesadmin/tools/qvm_template.py +++ b/qubesadmin/tools/qvm_template.py @@ -996,6 +996,7 @@ def install( package_hdrs = download(args, app, dl_list=dl_list, + path_override=args.cachedir, version_selector=version_selector) # Verify downloaded templates