Qubes/core-admin-client
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

qvm-template: Switch to namedtuples and other slight cleanup.

Browse Source
This commit is contained in:
WillyPillow 2020-08-01 02:24:29 +08:00
parent 3ada7af0eb
commit 233e411c2f
No known key found for this signature in database
GPG Key ID: 3839E194B1415A9C
1 changed files with 130 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

251
qubesadmin/tools/qvm_template.py
View File

@ -103,6 +103,50 @@ class VersionSelector(enum.Enum):
LATEST_LOWER = enum.auto() LATEST_LOWER = enum.auto()
LATEST_HIGHER = enum.auto() LATEST_HIGHER = enum.auto()
Template = collections.namedtuple('Template', [
'name',
'epoch',
'version',
'release',
'reponame',
'dlsize',
'buildtime',
'licence',
'url',
'summary',
'description'
])
DlEntry = collections.namedtuple('DlEntry', [
'evr',
'reponame',
'dlsize'
])
def query_local(vm):
return Template(
vm.features['template-name'],
vm.features['template-epoch'],
vm.features['template-version'],
vm.features['template-release'],
vm.features['template-reponame'],
vm.get_disk_utilization(),
vm.features['template-buildtime'],
vm.features['template-license'],
vm.features['template-url'],
vm.features['template-summary'],
vm.features['template-description'].replace('|', '\n'))
def query_local_evr(vm):
return (
vm.features['template-epoch'],
vm.features['template-version'],
vm.features['template-release'])
def is_managed_template(vm):
return 'template-name' in vm.features \
and vm.name == vm.features['template-name']
# NOTE: Verifying RPMs this way is prone to TOCTOU. This is okay for local # NOTE: Verifying RPMs this way is prone to TOCTOU. This is okay for local
# files, but may create problems if multiple instances of `qvm-template` are # files, but may create problems if multiple instances of `qvm-template` are
# downloading the same file, so a lock is needed in that case. # downloading the same file, so a lock is needed in that case.
@ -159,7 +203,7 @@ def install(args, app, version_selector=VersionSelector.LATEST,
try: try:
transaction_set = rpm.TransactionSet() transaction_set = rpm.TransactionSet()
rpm_list = [] rpm_list = [] # rpmfile, dlsize, reponame
for template in args.templates: for template in args.templates:
if template.endswith('.rpm'): if template.endswith('.rpm'):
if not os.path.exists(template): if not os.path.exists(template):
@ -172,25 +216,27 @@ def install(args, app, version_selector=VersionSelector.LATEST,
dl_list = get_dl_list(args, app, version_selector=version_selector) dl_list = get_dl_list(args, app, version_selector=version_selector)
dl_list_copy = dl_list.copy() dl_list_copy = dl_list.copy()
# Verify that the templates are not yet installed # Verify that the templates are not yet installed
for name, (ver, dlsize, reponame) in dl_list.items(): for name, entry in dl_list.items():
assert reponame != '@commandline' # Should be ensured by checks in repoquery
assert entry.reponame != '@commandline'
if not override_existing and name in app.domains: if not override_existing and name in app.domains:
print(('Template \'%s\' already installed, skipping...' print(('Template \'%s\' already installed, skipping...'
' (You may want to use the {reinstall,upgrade,downgrade}' ' (You may want to use the {reinstall,upgrade,downgrade}'
' operations.)') % name, file=sys.stderr) ' operations.)') % name, file=sys.stderr)
del dl_list_copy[name] del dl_list_copy[name]
else: else:
version_str = build_version_str(ver) version_str = build_version_str(entry.evr)
target_file = \ target_file = \
'%s%s-%s.rpm' % (PACKAGE_NAME_PREFIX, name, version_str) '%s%s-%s.rpm' % (PACKAGE_NAME_PREFIX, name, version_str)
rpm_list.append((os.path.join(args.cachedir, target_file), rpm_list.append((os.path.join(args.cachedir, target_file),
dlsize, reponame)) entry.dlsize, entry.reponame))
dl_list = dl_list_copy dl_list = dl_list_copy
download(args, app, path_override=args.cachedir, download(args, app, path_override=args.cachedir,
dl_list=dl_list, suffix=UNVERIFIED_SUFFIX, dl_list=dl_list, suffix=UNVERIFIED_SUFFIX,
version_selector=version_selector) version_selector=version_selector)
# Verify package and remove unverified suffix
for rpmfile, dlsize, reponame in rpm_list: for rpmfile, dlsize, reponame in rpm_list:
if reponame != '@commandline': if reponame != '@commandline':
path = rpmfile + UNVERIFIED_SUFFIX path = rpmfile + UNVERIFIED_SUFFIX
@ -201,6 +247,7 @@ def install(args, app, version_selector=VersionSelector.LATEST,
if reponame != '@commandline': if reponame != '@commandline':
os.rename(path, rpmfile) os.rename(path, rpmfile)
# Unpack and install
for rpmfile, dlsize, reponame in rpm_list: for rpmfile, dlsize, reponame in rpm_list:
with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target: with tempfile.TemporaryDirectory(dir=TEMP_DIR) as target:
package_hdr = get_package_hdr(rpmfile) package_hdr = get_package_hdr(rpmfile)
@ -229,10 +276,7 @@ def install(args, app, version_selector=VersionSelector.LATEST,
str(package_hdr[rpm.RPMTAG_EPOCHNUM]), str(package_hdr[rpm.RPMTAG_EPOCHNUM]),
package_hdr[rpm.RPMTAG_VERSION], package_hdr[rpm.RPMTAG_VERSION],
package_hdr[rpm.RPMTAG_RELEASE]) package_hdr[rpm.RPMTAG_RELEASE])
vm_evr = ( vm_evr = query_local_evr(vm)
vm.features['template-epoch'],
vm.features['template-version'],
vm.features['template-release'])
cmp_res = rpm.labelCompare(pkg_evr, vm_evr) cmp_res = rpm.labelCompare(pkg_evr, vm_evr)
if version_selector == VersionSelector.REINSTALL \ if version_selector == VersionSelector.REINSTALL \
and cmp_res != 0: and cmp_res != 0:
@ -311,6 +355,8 @@ def qrexec_popen(args, app, service, stdout=subprocess.PIPE, filter_esc=True):
def qrexec_payload(args, app, spec, refresh): def qrexec_payload(args, app, spec, refresh):
_ = app # unused _ = app # unused
# TODO: Check that spec != '---'
def check_newline(string, name): def check_newline(string, name):
if '\n' in string: if '\n' in string:
parser.error(f"Malformed {name}:" + parser.error(f"Malformed {name}:" +
@ -360,6 +406,8 @@ def qrexec_repoquery(args, app, spec='*', refresh=False):
entry = line.split('|') entry = line.split('|')
try: try:
# If there is an incorrect number of entries, raise an error # If there is an incorrect number of entries, raise an error
# Unpack manually instead of stuffing into `Template` right away
# so that it's easier to mutate stuff.
name, epoch, version, release, reponame, dlsize, \ name, epoch, version, release, reponame, dlsize, \
buildtime, licence, url, summary, description = entry buildtime, licence, url, summary, description = entry
@ -388,8 +436,8 @@ def qrexec_repoquery(args, app, spec='*', refresh=False):
if not is_match_spec(name, epoch, version, release, spec): if not is_match_spec(name, epoch, version, release, spec):
continue continue
result.append((name, epoch, version, release, reponame, dlsize, result.append(Template(name, epoch, version, release, reponame,
buildtime, licence, url, summary, description)) dlsize, buildtime, licence, url, summary, description))
except (TypeError, ValueError): except (TypeError, ValueError):
raise ConnectionError(("qrexec call 'qubes.TemplateSearch' failed:" raise ConnectionError(("qrexec call 'qubes.TemplateSearch' failed:"
" unexpected data format.")) " unexpected data format."))
@ -446,31 +494,27 @@ def list_templates(args, app, operation):
def append_list(data, status, install_time=None): def append_list(data, status, install_time=None):
_ = install_time # unused _ = install_time # unused
#pylint: disable=unused-variable version_str = build_version_str(
name, epoch, version, release, reponame, dlsize, \ (data.epoch, data.version, data.release))
buildtime, licence, url, summary, description = data tpl_list.append((status, data.name, version_str, data.reponame))
version_str = build_version_str((epoch, version, release))
tpl_list.append((status, name, version_str, reponame))
def append_info(data, status, install_time=None): def append_info(data, status, install_time=None):
name, epoch, version, release, reponame, dlsize, \ tpl_list.append((status, 'Name', ':', data.name))
buildtime, licence, url, summary, description = data tpl_list.append((status, 'Epoch', ':', data.epoch))
tpl_list.append((status, 'Name', ':', name)) tpl_list.append((status, 'Version', ':', data.version))
tpl_list.append((status, 'Epoch', ':', epoch)) tpl_list.append((status, 'Release', ':', data.release))
tpl_list.append((status, 'Version', ':', version))
tpl_list.append((status, 'Release', ':', release))
tpl_list.append((status, 'Size', ':', tpl_list.append((status, 'Size', ':',
qubesadmin.utils.size_to_human(dlsize))) qubesadmin.utils.size_to_human(data.dlsize)))
tpl_list.append((status, 'Repository', ':', reponame)) tpl_list.append((status, 'Repository', ':', data.reponame))
tpl_list.append((status, 'Buildtime', ':', str(buildtime))) tpl_list.append((status, 'Buildtime', ':', str(data.buildtime)))
if install_time: if install_time:
tpl_list.append((status, 'Install time', ':', str(install_time))) tpl_list.append((status, 'Install time', ':', str(install_time)))
tpl_list.append((status, 'URL', ':', url)) tpl_list.append((status, 'URL', ':', data.url))
tpl_list.append((status, 'License', ':', licence)) tpl_list.append((status, 'License', ':', data.licence))
tpl_list.append((status, 'Summary', ':', summary)) tpl_list.append((status, 'Summary', ':', data.summary))
# Only show "Description" for the first line # Only show "Description" for the first line
title = 'Description' title = 'Description'
for line in description.splitlines(): for line in data.description.splitlines():
tpl_list.append((status, title, ':', line)) tpl_list.append((status, title, ':', line))
title = '' title = ''
tpl_list.append((status, ' ', ' ', ' ')) # empty line tpl_list.append((status, ' ', ' ', ' ')) # empty line
@ -483,22 +527,7 @@ def list_templates(args, app, operation):
assert False and 'Unknown operation' assert False and 'Unknown operation'
def append_vm(vm, status): def append_vm(vm, status):
if vm.name == vm.features['template-name']: append(query_local(vm), status, vm.features['template-install-time'])
append((
vm.features['template-name'],
vm.features['template-epoch'],
vm.features['template-version'],
vm.features['template-release'],
vm.features['template-reponame'],
vm.get_disk_utilization(),
vm.features['template-buildtime'],
vm.features['template-license'],
vm.features['template-url'],
vm.features['template-summary'],
vm.features['template-description'].replace('|', '\n')
),
status,
vm.features['template-install-time'])
if not (args.installed or args.available or args.extras or args.upgrades): if not (args.installed or args.available or args.extras or args.upgrades):
args.all = True args.all = True
@ -514,13 +543,11 @@ def list_templates(args, app, operation):
if args.installed or args.all: if args.installed or args.all:
for vm in app.domains: for vm in app.domains:
if 'template-name' in vm.features: if is_managed_template(vm):
if not args.templates or \ if not args.templates or \
any(is_match_spec( any(is_match_spec(
vm.features['template-name'], vm.features['template-name'],
vm.features['template-epoch'], *query_local_evr(vm),
vm.features['template-version'],
vm.features['template-release'],
spec)[0] spec)[0]
for spec in args.templates): for spec in args.templates):
append_vm(vm, TemplateState.INSTALLED) append_vm(vm, TemplateState.INSTALLED)
@ -531,29 +558,23 @@ def list_templates(args, app, operation):
if args.extras: if args.extras:
remote = set() remote = set()
#pylint: disable=unused-variable for data in query_res:
for name, epoch, version, release, reponame, dlsize, \ remote.add(data.name)
buildtime, licence, url, summary, description in query_res:
remote.add(name)
for vm in app.domains: for vm in app.domains:
if 'template-name' in vm.features and \ if is_managed_template(vm) and \
vm.features['template-name'] not in remote: vm.features['template-name'] not in remote:
append_vm(vm, TemplateState.EXTRA) append_vm(vm, TemplateState.EXTRA)
if args.upgrades: if args.upgrades:
local = {} local = {}
for vm in app.domains: for vm in app.domains:
if 'template-name' in vm.features: if is_managed_template(vm):
local[vm.features['template-name']] = ( local[vm.features['template-name']] = query_local_evr(vm)
vm.features['template-epoch'], for entry in query_res:
vm.features['template-version'], if entry.name in local:
vm.features['template-release']) if rpm.labelCompare(local[entry.name],
for data in query_res: (entry.epoch, entry.version, entry.release)) < 0:
name, epoch, version, release, reponame, dlsize, \ append(entry, TemplateState.UPGRADABLE)
buildtime, licence, url, summary, description = data
if name in local:
if rpm.labelCompare(local[name], (epoch, version, release)) < 0:
append(data, TemplateState.UPGRADABLE)
if len(tpl_list) == 0: if len(tpl_list) == 0:
parser.error('No matching templates to list') parser.error('No matching templates to list')
@ -566,24 +587,12 @@ def search(args, app):
# Search in both installed and available templates # Search in both installed and available templates
query_res = qrexec_repoquery(args, app) query_res = qrexec_repoquery(args, app)
for vm in app.domains: for vm in app.domains:
if 'template-name' in vm.features: if is_managed_template(vm):
query_res.append(( query_res.append(query_local(vm))
vm.features['template-name'],
vm.features['template-epoch'],
vm.features['template-version'],
vm.features['template-release'],
vm.features['template-reponame'],
vm.get_disk_utilization(),
datetime.datetime.fromisoformat(
vm.features['template-buildtime']),
vm.features['template-license'],
vm.features['template-url'],
vm.features['template-summary'],
vm.features['template-description']))
# Get latest version for each template # Get latest version for each template
query_res_tmp = [] query_res_tmp = []
for name, grp in itertools.groupby(query_res, lambda x: x[0]): for name, grp in itertools.groupby(sorted(query_res), lambda x: x[0]):
def compare(lhs, rhs): def compare(lhs, rhs):
return lhs if rpm.labelCompare(lhs[1:4], rhs[1:4]) < 0 else rhs return lhs if rpm.labelCompare(lhs[1:4], rhs[1:4]) < 0 else rhs
query_res_tmp.append(functools.reduce(compare, grp)) query_res_tmp.append(functools.reduce(compare, grp))
@ -605,14 +614,12 @@ def search(args, app):
search_res = collections.defaultdict(list) search_res = collections.defaultdict(list)
for keyword in args.templates: for keyword in args.templates:
#pylint: disable=unused-variable for idx, entry in enumerate(query_res):
for idx, (name, epoch, version, release, reponame, dlsize, \ needles = \
buildtime, licence, url, summary, description) \ [(entry.name, WEIGHT_NAME), (entry.summary, WEIGHT_SUMMARY)]
in enumerate(query_res):
needles = [(name, WEIGHT_NAME), (summary, WEIGHT_SUMMARY)]
if args.all: if args.all:
needles += \ needles += [(entry.description, WEIGHT_DESCRIPTION),
[(description, WEIGHT_DESCRIPTION), (url, WEIGHT_URL)] (entry.url, WEIGHT_URL)]
for key, weight in needles: for key, weight in needles:
if fnmatch.fnmatch(key, '*' + keyword + '*'): if fnmatch.fnmatch(key, '*' + keyword + '*'):
exact = keyword == key exact = keyword == key
@ -620,7 +627,6 @@ def search(args, app):
weight = WEIGHT_NAME_EXACT weight = WEIGHT_NAME_EXACT
search_res[idx].append((weight, keyword, exact)) search_res[idx].append((weight, keyword, exact))
# Requires changes to the qrexec call qubes.TemplateSearch
if not args.all: if not args.all:
keywords = set(args.templates) keywords = set(args.templates)
idxs = list(search_res.keys()) idxs = list(search_res.keys())
@ -638,9 +644,6 @@ def search(args, app):
search_res = sorted(search_res.items(), key=key_func) search_res = sorted(search_res.items(), key=key_func)
def gen_header(idx, needles): def gen_header(idx, needles):
#pylint: disable=unused-variable
name, epoch, version, release, reponame, dlsize, \
buildtime, licence, url, summary, description = query_res[idx]
fields = [] fields = []
weight_types = set(x[0] for x in needles) weight_types = set(x[0] for x in needles)
for weight, field in WEIGHT_TO_FIELD: for weight, field in WEIGHT_TO_FIELD:
@ -659,9 +662,7 @@ def search(args, app):
last_header = cur_header last_header = cur_header
# XXX: The style is different from that of DNF # XXX: The style is different from that of DNF
print('===', cur_header, '===') print('===', cur_header, '===')
name, epoch, version, release, reponame, dlsize, \ print(query_res[idx].name, ':', query_res[idx].summary)
buildtime, licence, url, summary, description = query_res[idx]
print(name, ':', summary)
def get_dl_list(args, app, version_selector=VersionSelector.LATEST): def get_dl_list(args, app, version_selector=VersionSelector.LATEST):
full_candid = {} full_candid = {}
@ -677,40 +678,38 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST):
query_res = qrexec_repoquery(args, app, PACKAGE_NAME_PREFIX + template) query_res = qrexec_repoquery(args, app, PACKAGE_NAME_PREFIX + template)
# We only select one package for each distinct package name # We only select one package for each distinct package name
#pylint: disable=unused-variable # TODO: Check local VM is managed by qvm-template
for name, epoch, version, release, reponame, dlsize, \ for entry in query_res:
buildtime, licence, url, summary, description in query_res: ver = (entry.epoch, entry.version, entry.release)
ver = (epoch, version, release) insert = False
if version_selector == VersionSelector.LATEST: if version_selector == VersionSelector.LATEST:
if name not in candid \ if entry.name not in candid \
or rpm.labelCompare(candid[name][0], ver) < 0: or rpm.labelCompare(candid[entry.name][0], ver) < 0:
candid[name] = (ver, dlsize, reponame) insert = True
elif version_selector == VersionSelector.REINSTALL: elif version_selector == VersionSelector.REINSTALL:
if name not in app.domains: if entry.name not in app.domains:
parser.error("Template '%s' not already installed." % name) parser.error(
vm = app.domains[name] "Template '%s' not already installed." % entry.name)
cur_ver = ( vm = app.domains[entry.name]
vm.features['template-epoch'], cur_ver = query_local_evr(vm)
vm.features['template-version'],
vm.features['template-release'])
if rpm.labelCompare(ver, cur_ver) == 0: if rpm.labelCompare(ver, cur_ver) == 0:
candid[name] = (ver, dlsize, reponame) insert = True
elif version_selector in [VersionSelector.LATEST_LOWER, elif version_selector in [VersionSelector.LATEST_LOWER,
VersionSelector.LATEST_HIGHER]: VersionSelector.LATEST_HIGHER]:
if name not in app.domains: if entry.name not in app.domains:
parser.error("Template '%s' not already installed." % name) parser.error(
vm = app.domains[name] "Template '%s' not already installed." % entry.name)
cur_ver = ( vm = app.domains[entry.name]
vm.features['template-epoch'], cur_ver = query_local_evr(vm)
vm.features['template-version'],
vm.features['template-release'])
cmp_res = -1 \ cmp_res = -1 \
if version_selector == VersionSelector.LATEST_LOWER \ if version_selector == VersionSelector.LATEST_LOWER \
else 1 else 1
if rpm.labelCompare(ver, cur_ver) == cmp_res: if rpm.labelCompare(ver, cur_ver) == cmp_res:
if name not in candid \ if entry.name not in candid \
or rpm.labelCompare(candid[name][0], ver) < 0: or rpm.labelCompare(candid[entry.name][0], ver) < 0:
candid[name] = (ver, dlsize, reponame) insert = True
if insert:
candid[entry.name] = DlEntry(ver, entry.reponame, entry.dlsize)
# XXX: As it's possible to include version information in `template` # XXX: As it's possible to include version information in `template`
# Perhaps the messages can be improved # Perhaps the messages can be improved
@ -731,10 +730,10 @@ def get_dl_list(args, app, version_selector=VersionSelector.LATEST):
file=sys.stderr) file=sys.stderr)
# Merge & choose the template with the highest version # Merge & choose the template with the highest version
for name, (ver, dlsize, reponame) in candid.items(): for name, entry in candid.items():
if name not in full_candid \ if name not in full_candid \
or rpm.labelCompare(full_candid[name][0], ver) < 0: or rpm.labelCompare(full_candid[name].evr, entry.evr) < 0:
full_candid[name] = (ver, dlsize, reponame) full_candid[name] = entry
return candid return candid
@ -744,9 +743,8 @@ def download(args, app, path_override=None,
dl_list = get_dl_list(args, app, version_selector=version_selector) dl_list = get_dl_list(args, app, version_selector=version_selector)
path = path_override if path_override is not None else args.downloaddir path = path_override if path_override is not None else args.downloaddir
for name, (ver, dlsize, reponame) in dl_list.items(): for name, entry in dl_list.items():
_ = reponame # unused version_str = build_version_str(entry.evr)
version_str = build_version_str(ver)
spec = PACKAGE_NAME_PREFIX + name + '-' + version_str spec = PACKAGE_NAME_PREFIX + name + '-' + version_str
target = os.path.join(path, '%s.rpm' % spec) target = os.path.join(path, '%s.rpm' % spec)
target_suffix = target + suffix target_suffix = target + suffix
@ -763,7 +761,8 @@ def download(args, app, path_override=None,
done = False done = False
for attempt in range(args.retries): for attempt in range(args.retries):
try: try:
qrexec_download(args, app, spec, target_suffix, dlsize) qrexec_download(args, app, spec, target_suffix,
entry.dlsize)
done = True done = True
break break
except ConnectionError: except ConnectionError: