From a982e1e538d270dd7e942732fdd83922fe4fbe46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Pierret=20=28fepitre=29?= Date: Sat, 10 Aug 2019 19:03:17 +0200 Subject: [PATCH 1/5] qubesadmin: make PEP8 happy --- qubesadmin/app.py | 150 +++++++++++++++++++++++----------------------- 1 file changed, 76 insertions(+), 74 deletions(-) diff --git a/qubesadmin/app.py b/qubesadmin/app.py index de16cd4..61ceb75 100644 --- a/qubesadmin/app.py +++ b/qubesadmin/app.py @@ -19,9 +19,9 @@ # with this program; if not, see . -''' +""" Main Qubes() class and related classes. -''' +""" import os import shlex import socket @@ -40,19 +40,21 @@ import qubesadmin.config BUF_SIZE = 4096 + class VMCollection(object): - '''Collection of VMs objects''' + """Collection of VMs objects""" + def __init__(self, app): self.app = app self._vm_list = None self._vm_objects = {} def clear_cache(self): - '''Clear cached list of VMs''' + """Clear cached list of VMs""" self._vm_list = None def refresh_cache(self, force=False): - '''Refresh cached list of VMs''' + """Refresh cached list of VMs""" if not force and self._vm_list is not None: return vm_list_data = self.app.qubesd_call( @@ -90,10 +92,10 @@ class VMCollection(object): return self.get_blind(item) def get_blind(self, item): - ''' + """ Get a vm without downloading the list and checking if exists - ''' + """ if item not in self._vm_objects: cls = qubesadmin.vm.QubesVM # provide class name to constructor, if already cached (which can be @@ -121,23 +123,23 @@ class VMCollection(object): yield self[vm] def keys(self): - '''Get list of VM names.''' + """Get list of VM names.""" self.refresh_cache() return self._vm_list.keys() def values(self): - '''Get list of VM objects.''' + """Get list of VM objects.""" self.refresh_cache() return [self[name] for name in self._vm_list] class QubesBase(qubesadmin.base.PropertyHolder): - '''Main Qubes application. + """Main Qubes application. This is a base abstract class, don't use it directly. Use specialized class in py:class:`qubesadmin.Qubes` instead, which points at :py:class:`QubesLocal` or :py:class:`QubesRemote`. - ''' + """ #: domains (VMs) collection domains = None @@ -164,11 +166,11 @@ class QubesBase(qubesadmin.base.PropertyHolder): self.log = logging.getLogger('app') def _refresh_pool_drivers(self): - ''' + """ Refresh cached storage pool drivers and their parameters. :return: None - ''' + """ if self._pool_drivers is None: pool_drivers_data = self.qubesd_call( 'dom0', 'admin.pool.ListDrivers', None, None) @@ -183,40 +185,40 @@ class QubesBase(qubesadmin.base.PropertyHolder): @property def pool_drivers(self): - ''' Available storage pool drivers ''' + """ Available storage pool drivers """ self._refresh_pool_drivers() return self._pool_drivers.keys() def pool_driver_parameters(self, driver): - ''' Parameters to initialize storage pool using given driver ''' + """ Parameters to initialize storage pool using given driver """ self._refresh_pool_drivers() return self._pool_drivers[driver] def add_pool(self, name, driver, **kwargs): - ''' Add a storage pool to config + """ Add a storage pool to config :param name: name of storage pool to create :param driver: driver to use, see :py:meth:`pool_drivers` for available drivers :param kwargs: configuration parameters for storage pool, see :py:meth:`pool_driver_parameters` for a list - ''' + """ # sort parameters only to ease testing, not required by API payload = 'name={}\n'.format(name) + \ ''.join('{}={}\n'.format(key, value) - for key, value in sorted(kwargs.items())) + for key, value in sorted(kwargs.items())) self.qubesd_call('dom0', 'admin.pool.Add', driver, - payload.encode('utf-8')) + payload.encode('utf-8')) def remove_pool(self, name): - ''' Remove a storage pool ''' + """ Remove a storage pool """ self.qubesd_call('dom0', 'admin.pool.Remove', name, None) def get_label(self, label): - '''Get label as identified by index or name + """Get label as identified by index or name :throws KeyError: when label is not found - ''' + """ # first search for name, verbatim try: @@ -233,19 +235,19 @@ class QubesBase(qubesadmin.base.PropertyHolder): @staticmethod def get_vm_class(clsname): - '''Find the class for a domain. + """Find the class for a domain. Compatibility function, client tools use str to identify domain classes. :param str clsname: name of the class :return str: class - ''' + """ return clsname def add_new_vm(self, cls, name, label, template=None, pool=None, - pools=None): - '''Create new Virtual Machine + pools=None): + """Create new Virtual Machine Example usage with custom storage pools: @@ -264,7 +266,7 @@ class QubesBase(qubesadmin.base.PropertyHolder): :param dict pools: storage pool for specific volumes :return new VM object - ''' + """ if not isinstance(cls, str): cls = cls.__name__ @@ -284,18 +286,18 @@ class QubesBase(qubesadmin.base.PropertyHolder): method_prefix = 'admin.vm.CreateInPool.' if pools: payload += ''.join(' pool:{}={}'.format(vol, str(pool)) - for vol, pool in sorted(pools.items())) + for vol, pool in sorted(pools.items())) method_prefix = 'admin.vm.CreateInPool.' self.qubesd_call('dom0', method_prefix + cls, template, - payload.encode('utf-8')) + payload.encode('utf-8')) self.domains.clear_cache() return self.domains[name] - def clone_vm(self, src_vm, new_name, new_cls=None, - pool=None, pools=None, ignore_errors=False, ignore_volumes=None): - '''Clone Virtual Machine + def clone_vm(self, src_vm, new_name, new_cls=None, pool=None, pools=None, + ignore_errors=False, ignore_volumes=None): + """Clone Virtual Machine Example usage with custom storage pools: @@ -317,7 +319,7 @@ class QubesBase(qubesadmin.base.PropertyHolder): like 'private' or 'root' :return new VM object - ''' + """ if pool and pools: raise ValueError('only one of pool= and pools= can be used') @@ -343,7 +345,7 @@ class QubesBase(qubesadmin.base.PropertyHolder): if ignore_volumes and volume.name in ignore_volumes: continue default_pool = getattr(self.app, 'default_pool_' + volume.name, - volume.pool) + volume.pool) if default_pool != volume.pool: if pools is None: pools = {} @@ -356,11 +358,11 @@ class QubesBase(qubesadmin.base.PropertyHolder): method_prefix = 'admin.vm.CreateInPool.' if pools: payload += ''.join(' pool:{}={}'.format(vol, str(pool)) - for vol, pool in sorted(pools.items())) + for vol, pool in sorted(pools.items())) method_prefix = 'admin.vm.CreateInPool.' self.qubesd_call('dom0', method_prefix + new_cls, template, - payload.encode('utf-8')) + payload.encode('utf-8')) self.domains.clear_cache() dst_vm = self.domains[new_name] @@ -369,7 +371,7 @@ class QubesBase(qubesadmin.base.PropertyHolder): for prop in src_vm.property_list(): # handled by admin.vm.Create call if prop in ('name', 'qid', 'template', 'label', 'uuid', - 'installed_by_rpm'): + 'installed_by_rpm'): continue if src_vm.property_is_default(prop): continue @@ -414,7 +416,7 @@ class QubesBase(qubesadmin.base.PropertyHolder): # FIXME: convert to qrexec calls to dom0/GUI VM appmenus_cmd = \ ['qvm-appmenus', '--init', '--update', - '--source', src_vm.name, dst_vm.name] + '--source', src_vm.name, dst_vm.name] subprocess.check_output(appmenus_cmd, stderr=subprocess.STDOUT) except OSError: # this file needs to be python 2.7 compatible, @@ -425,7 +427,7 @@ class QubesBase(qubesadmin.base.PropertyHolder): 'Failed to clone appmenus') except subprocess.CalledProcessError as e: self.log.error('Failed to clone appmenus: %s', - e.output.decode()) + e.output.decode()) if not ignore_errors: raise qubesadmin.exc.QubesException( 'Failed to clone appmenus') @@ -453,8 +455,8 @@ class QubesBase(qubesadmin.base.PropertyHolder): return dst_vm def qubesd_call(self, dest, method, arg=None, payload=None, - payload_stream=None): - ''' + payload_stream=None): + """ Execute Admin API method. Only one of `payload` and `payload_stream` can be specified. @@ -467,14 +469,14 @@ class QubesBase(qubesadmin.base.PropertyHolder): :return: Data returned by qubesd (string) .. warning:: *payload_stream* will get closed by this function - ''' + """ raise NotImplementedError( 'qubesd_call not implemented in QubesBase class; use specialized ' 'class: qubesadmin.Qubes()') def run_service(self, dest, service, filter_esc=False, user=None, - localcmd=None, wait=True, **kwargs): - '''Run qrexec service in a given destination + localcmd=None, wait=True, **kwargs): + """Run qrexec service in a given destination *kwargs* are passed verbatim to :py:meth:`subprocess.Popen`. @@ -485,24 +487,25 @@ class QubesBase(qubesadmin.base.PropertyHolder): emulator :param str user: username to run service as :param str localcmd: Command to connect stdin/stdout to + :param bool wait: Wait service run :rtype: subprocess.Popen - ''' + """ raise NotImplementedError( 'run_service not implemented in QubesBase class; use specialized ' 'class: qubesadmin.Qubes()') class QubesLocal(QubesBase): - '''Application object communicating through local socket. + """Application object communicating through local socket. Used when running in dom0. - ''' + """ qubesd_connection_type = 'socket' def qubesd_call(self, dest, method, arg=None, payload=None, - payload_stream=None): - ''' + payload_stream=None): + """ Execute Admin API method. Only one of `payload` and `payload_stream` can be specified. @@ -515,7 +518,7 @@ class QubesLocal(QubesBase): :return: Data returned by qubesd (string) .. warning:: *payload_stream* will get closed by this function - ''' + """ if payload and payload_stream: raise ValueError( 'Only one of payload and payload_stream can be used') @@ -530,11 +533,11 @@ class QubesLocal(QubesBase): raise qubesadmin.exc.QubesDaemonCommunicationError( '{} not found'.format(method_path)) command = ['env', 'QREXEC_REMOTE_DOMAIN=dom0', - 'QREXEC_REQUESTED_TARGET=' + dest, method_path, arg] + 'QREXEC_REQUESTED_TARGET=' + dest, method_path, arg] if os.getuid() != 0: command.insert(0, 'sudo') proc = subprocess.Popen(command, stdin=payload_stream, - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) payload_stream.close() (return_data, _) = proc.communicate() return self._parse_qubesd_response(return_data) @@ -561,8 +564,8 @@ class QubesLocal(QubesBase): return self._parse_qubesd_response(return_data) def run_service(self, dest, service, filter_esc=False, user=None, - localcmd=None, wait=True, **kwargs): - '''Run qrexec service in a given destination + localcmd=None, wait=True, **kwargs): + """Run qrexec service in a given destination :param str dest: Destination - may be a VM name or empty string for default (for a given service) @@ -572,9 +575,8 @@ class QubesLocal(QubesBase): :param str user: username to run service as :param str localcmd: Command to connect stdin/stdout to :param bool wait: wait for remote process to finish - :param int connect_timeout: qrexec client connection timeout :rtype: subprocess.Popen - ''' + """ if not dest: raise ValueError('Empty destination name allowed only from a VM') @@ -600,23 +602,23 @@ class QubesLocal(QubesBase): kwargs.setdefault('stdin', subprocess.PIPE) kwargs.setdefault('stdout', subprocess.PIPE) kwargs.setdefault('stderr', subprocess.PIPE) - proc = subprocess.Popen([qubesadmin.config.QREXEC_CLIENT] + - qrexec_opts + ['{}:QUBESRPC {} dom0'.format(user, service)], - **kwargs) + proc = subprocess.Popen( + [qubesadmin.config.QREXEC_CLIENT] + qrexec_opts + [ + '{}:QUBESRPC {} dom0'.format(user, service)], **kwargs) return proc class QubesRemote(QubesBase): - '''Application object communicating through qrexec services. + """Application object communicating through qrexec services. Used when running in VM. - ''' + """ qubesd_connection_type = 'qrexec' def qubesd_call(self, dest, method, arg=None, payload=None, - payload_stream=None): - ''' + payload_stream=None): + """ Execute Admin API method. Only one of `payload` and `payload_stream` can be specified. @@ -629,7 +631,7 @@ class QubesRemote(QubesBase): :return: Data returned by qubesd (string) .. warning:: *payload_stream* will get closed by this function - ''' + """ if payload and payload_stream: raise ValueError( 'Only one of payload and payload_stream can be used') @@ -637,10 +639,10 @@ class QubesRemote(QubesBase): if arg is not None: service_name += '+' + arg p = subprocess.Popen([qubesadmin.config.QREXEC_CLIENT_VM, - dest, service_name], - stdin=(payload_stream or subprocess.PIPE), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + dest, service_name], + stdin=(payload_stream or subprocess.PIPE), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) if payload_stream is not None: payload_stream.close() (stdout, stderr) = p.communicate(payload) @@ -651,8 +653,8 @@ class QubesRemote(QubesBase): return self._parse_qubesd_response(stdout) def run_service(self, dest, service, filter_esc=False, user=None, - localcmd=None, wait=True, **kwargs): - '''Run qrexec service in a given destination + localcmd=None, wait=True, **kwargs): + """Run qrexec service in a given destination :param str dest: Destination - may be a VM name or empty string for default (for a given service) @@ -663,7 +665,7 @@ class QubesRemote(QubesBase): :param str localcmd: Command to connect stdin/stdout to :param bool wait: wait for process to finish :rtype: subprocess.Popen - ''' + """ if filter_esc: raise NotImplementedError( 'filter_esc not implemented for calls from VM') @@ -685,7 +687,7 @@ class QubesRemote(QubesBase): kwargs.setdefault('stdin', subprocess.PIPE) kwargs.setdefault('stdout', subprocess.PIPE) kwargs.setdefault('stderr', subprocess.PIPE) - proc = subprocess.Popen([qubesadmin.config.QREXEC_CLIENT_VM, - dest or '', service] + (shlex.split(localcmd) if localcmd else []), - **kwargs) + proc = subprocess.Popen( + [qubesadmin.config.QREXEC_CLIENT_VM, dest or '', service] + ( + shlex.split(localcmd) if localcmd else []), **kwargs) return proc From 3dce4e974270d47fff67846d7659272ca9589b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Pierret=20=28fepitre=29?= Date: Sat, 10 Aug 2019 22:08:28 +0200 Subject: [PATCH 2/5] qubesadmin: define methods list_vmclass and list_devicesclass Adapt also previous direct calls of qubesdb QubesOS/qubes-issues#5213 --- qubesadmin/app.py | 12 ++++++++++++ qubesadmin/devices.py | 14 ++------------ qubesadmin/tools/qvm_create.py | 5 ++--- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/qubesadmin/app.py b/qubesadmin/app.py index 61ceb75..e1469d6 100644 --- a/qubesadmin/app.py +++ b/qubesadmin/app.py @@ -165,6 +165,18 @@ class QubesBase(qubesadmin.base.PropertyHolder): self._pool_drivers = None self.log = logging.getLogger('app') + def list_vmclass(self): + """Call Qubesd in order to obtain the vm classes list""" + vmclass = self.qubesd_call('dom0', 'admin.vmclass.List')\ + .decode().splitlines() + return sorted(vmclass) + + def list_deviceclass(self): + """Call Qubesd in order to obtain the device classes list""" + deviceclasses = self.qubesd_call('dom0', 'admin.deviceclass.List')\ + .decode().splitlines() + return sorted(deviceclasses) + def _refresh_pool_drivers(self): """ Refresh cached storage pool drivers and their parameters. diff --git a/qubesadmin/devices.py b/qubesadmin/devices.py index 646ef2d..c09e1ce 100644 --- a/qubesadmin/devices.py +++ b/qubesadmin/devices.py @@ -303,17 +303,7 @@ class DeviceManager(dict): return self[key] def __iter__(self): - return iter(self._get_device_classes()) + return iter(self._vm.app.list_deviceclass()) def keys(self): - return self._get_device_classes() - - def _get_device_classes(self): - """Function used to call Qubesd in order to obtain - the device classes list - """ - device_classes = \ - self._vm.app.qubesd_call('dom0', 'admin.deviceclass.List').decode() - device_classes = sorted(device_classes.splitlines()) - - return device_classes + return self._vm.app.list_deviceclass() diff --git a/qubesadmin/tools/qvm_create.py b/qubesadmin/tools/qvm_create.py index b7ded60..9dbcbe1 100644 --- a/qubesadmin/tools/qvm_create.py +++ b/qubesadmin/tools/qvm_create.py @@ -103,9 +103,8 @@ def main(args=None, app=None): args = parser.parse_args(args, app=app) if args.help_classes: - vm_classes = args.app.qubesd_call('dom0', 'admin.vmclass.List').decode() - vm_classes = vm_classes.splitlines() - print('\n'.join(sorted(vm_classes))) + vm_classes = args.app.list_vmclass() + print('\n'.join(vm_classes)) return 0 pools = {} From 41c6fbed72b349e6bc61d09897a9e5bb4304c65f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Pierret=20=28fepitre=29?= Date: Sat, 10 Aug 2019 22:10:31 +0200 Subject: [PATCH 3/5] qvm-device: handle 'list-device-classes' and 'list-classes' QubesOS/qubes-issues#5213 --- qubesadmin/qubesadmin | 1 + qubesadmin/tools/qvm_device.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 120000 qubesadmin/qubesadmin diff --git a/qubesadmin/qubesadmin b/qubesadmin/qubesadmin new file mode 120000 index 0000000..d8da336 --- /dev/null +++ b/qubesadmin/qubesadmin @@ -0,0 +1 @@ +/home/user/qubes-builder/qubes-src/core-admin-client/qubesadmin \ No newline at end of file diff --git a/qubesadmin/tools/qvm_device.py b/qubesadmin/tools/qvm_device.py index 0121b5f..c5a880d 100644 --- a/qubesadmin/tools/qvm_device.py +++ b/qubesadmin/tools/qvm_device.py @@ -263,6 +263,9 @@ def get_parser(device_class=None): attach_parser.set_defaults(func=attach_device) detach_parser.set_defaults(func=detach_device) + parser.add_argument('--list-device-classes', action='store_true', + default=False) + return parser @@ -272,7 +275,13 @@ def main(args=None, app=None): devclass = None if basename.startswith('qvm-') and basename != 'qvm-device': devclass = basename[4:] + args = get_parser(devclass).parse_args(args, app=app) + + if args.list_device_classes: + print('\n'.join(qubesadmin.Qubes().list_deviceclass())) + return 0 + try: args.func(args) except qubesadmin.exc.QubesException as e: @@ -282,4 +291,10 @@ def main(args=None, app=None): if __name__ == '__main__': + # Special treatment for '--list-device-classes' (alias --list-classes) + curr_action = sys.argv[1:] + if set(curr_action).intersection( + {'--list-device-classes', '--list-classes'}): + sys.exit(main(args=['', '--list-device-classes'])) + sys.exit(main()) From 8639034bb6e1256c41cb9211f48ef23cdd2ad8d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Pierret=20=28fepitre=29?= Date: Sat, 24 Aug 2019 13:29:07 +0200 Subject: [PATCH 4/5] qvm-device: prevent parser allowing abbreviations --- qubesadmin/tools/qvm_device.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qubesadmin/tools/qvm_device.py b/qubesadmin/tools/qvm_device.py index c5a880d..707710c 100644 --- a/qubesadmin/tools/qvm_device.py +++ b/qubesadmin/tools/qvm_device.py @@ -211,6 +211,7 @@ def get_parser(device_class=None): want_app=True) parser.register('action', 'parsers', qubesadmin.tools.AliasedSubParsersAction) + parser.allow_abbrev = False if device_class: parser.add_argument('devclass', const=device_class, action='store_const', From d96db420ba639561dd869695098050e5720b704d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Pierret=20=28fepitre=29?= Date: Sat, 24 Aug 2019 13:33:29 +0200 Subject: [PATCH 5/5] qvm-device: add manpage entry --- doc/manpages/qvm-device.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manpages/qvm-device.rst b/doc/manpages/qvm-device.rst index 8b957f5..733f5b9 100644 --- a/doc/manpages/qvm-device.rst +++ b/doc/manpages/qvm-device.rst @@ -28,6 +28,10 @@ Options decrease verbosity +.. option:: --list-device-classes + + list device classes + Commands ======== @@ -122,3 +126,4 @@ Authors | Joanna Rutkowska | Rafal Wojtczuk | Marek Marczykowski +| Frédéric Pierret