Merge branch 'fc31'

* fc31:
  rpm: switch deps to python3-setuptools on CentOS too
  debian: switch to python3
  Use spaces in xdg-icon script
  Convert other scripts to python3
  Convert qubesagent module to python3
  Minor codestyle fix in qubesadmin/firewall.py
  Require python setuptools
  Update python2 dependencies to python3 and clean deprecated requirements

Makefile

@@ -11,7 +11,7 @@ BINDIR ?= /usr/bin
 LIBDIR ?= /usr/lib
 SYSLIBDIR ?= /lib
 
-PYTHON ?= /usr/bin/python2
+PYTHON ?= /usr/bin/python3
 PYTHON_SITEARCH = $(shell python2 -c 'import distutils.sysconfig; print distutils.sysconfig.get_python_lib(1)')
 PYTHON2_SITELIB = $(shell python2 -c 'import distutils.sysconfig; print distutils.sysconfig.get_python_lib()')
 PYTHON3_SITELIB = $(shell python3 -c 'import distutils.sysconfig; print(distutils.sysconfig.get_python_lib())')

archlinux/PKGBUILD

@@ -61,10 +61,10 @@ build() {
 # * core systemd services and drop-ins
 # * basic network functionality (setting IP address, DNS, default gateway)
 package_qubes-vm-core() {
-    depends=("qubes-vm-utils>=3.1.3" python2 python2-xdg ethtool ntp net-tools
+    depends=("qubes-vm-utils>=3.1.3" python2 python3-xdg ethtool ntp net-tools
              gnome-packagekit imagemagick fakeroot notification-daemon dconf
-             zenity qubes-libvchan "qubes-db-vm>=3.2.1" haveged python2-gobject
+             zenity qubes-libvchan "qubes-db-vm>=3.2.1" haveged python3-gobject
-             python2-dbus xdg-utils notification-daemon gawk sed procps-ng librsvg
+             python3-dbus xdg-utils notification-daemon gawk sed procps-ng librsvg
              socat
              )
     optdepends=(gnome-keyring gnome-settings-daemon python2-nautilus gpk-update-viewer qubes-vm-networking qubes-vm-keyring)
@@ -74,7 +74,7 @@ package_qubes-vm-core() {
     # shellcheck disable=SC2154
     make -C qrexec install DESTDIR="$pkgdir" SBINDIR=/usr/bin LIBDIR=/usr/lib SYSLIBDIR=/usr/lib
 
-    PYTHON=python2 make install-corevm DESTDIR="$pkgdir" SBINDIR=/usr/bin LIBDIR=/usr/lib SYSLIBDIR=/usr/lib SYSTEM_DROPIN_DIR=/usr/lib/systemd/system USER_DROPIN_DIR=/usr/lib/systemd/user DIST=archlinux
+    make install-corevm DESTDIR="$pkgdir" SBINDIR=/usr/bin LIBDIR=/usr/lib SYSLIBDIR=/usr/lib SYSTEM_DROPIN_DIR=/usr/lib/systemd/system USER_DROPIN_DIR=/usr/lib/systemd/user DIST=archlinux
 
     # Remove things non wanted in archlinux
     rm -r "$pkgdir/etc/yum"*
@@ -122,13 +122,13 @@ EOF
 #
 package_qubes-vm-networking() {
     pkgdesc="Qubes OS tools allowing to use a Qubes VM as a NetVM/ProxyVM"
-    depends=(qubes-vm-core "qubes-vm-utils>=3.1.3" python2 ethtool net-tools
+    depends=(qubes-vm-core "qubes-vm-utils>=3.1.3" python3 ethtool net-tools
              "qubes-db-vm>=3.2.1" networkmanager iptables tinyproxy nftables
              )
     install=PKGBUILD-networking.install
 
     # shellcheck disable=SC2154
-    PYTHON=python2 make install-netvm DESTDIR="$pkgdir" SBINDIR=/usr/bin LIBDIR=/usr/lib SYSLIBDIR=/usr/lib SYSTEM_DROPIN_DIR=/usr/lib/systemd/system USER_DROPIN_DIR=/usr/lib/systemd/user DIST=archlinux
+    make install-netvm DESTDIR="$pkgdir" SBINDIR=/usr/bin LIBDIR=/usr/lib SYSLIBDIR=/usr/lib SYSTEM_DROPIN_DIR=/usr/lib/systemd/system USER_DROPIN_DIR=/usr/lib/systemd/user DIST=archlinux
 
 }
 

debian/control

@@ -7,7 +7,7 @@ Build-Depends:
     libqubes-rpc-filecopy-dev (>= 3.1.3),
     libvchan-xen-dev,
     python,
-    python-setuptools,
+    python3-setuptools,
     debhelper,
     quilt,
     libxen-dev,
@@ -39,12 +39,12 @@ Depends:
     procps,
     util-linux,
     e2fsprogs,
-    python2.7,
+    python3-daemon,
-    python-daemon,
+    python3-distutils,
-    python-qubesdb,
+    python3-qubesdb,
-    python-gi,
+    python3-gi,
-    python-xdg,
+    python3-xdg,
-    python-dbus,
+    python3-dbus,
     qubes-utils (>= 3.1.3),
     qubes-core-qrexec,
     qubesdb-vm,
@@ -54,7 +54,7 @@ Depends:
     xen-utils-common,
     xen-utils-guest,
     xenstore-utils,
-    ${python:Depends},
+    ${python3:Depends},
     ${shlibs:Depends},
     ${misc:Depends}
 Recommends:
@@ -128,7 +128,7 @@ Depends:
     socat,
     tinyproxy,
     iproute2,
-    ${python:Depends},
+    ${python3:Depends},
     ${misc:Depends}
 Suggests:
     nftables,

debian/qubes-core-agent.install

@@ -107,8 +107,8 @@ usr/bin/qvm-connect-tcp
 usr/bin/qvm-console
 usr/bin/qvm-sync-clock
 usr/bin/xenstore-watch-qubes
-usr/lib/python2.7/dist-packages/qubesagent-*.egg-info/*
+usr/lib/python3/dist-packages/qubesagent-*.egg-info/*
-usr/lib/python2.7/dist-packages/qubesagent/*
+usr/lib/python3/dist-packages/qubesagent/*
 usr/lib/qubes-bind-dirs.d/30_cron.conf
 usr/lib/qubes/close-window
 usr/lib/qubes/init/bind-dirs.sh

debian/rules

@@ -9,7 +9,7 @@ include /usr/share/dpkg/default.mk
 export DESTDIR=$(shell pwd)/debian/tmp
 
 %:
-	dh $@ --with systemd,python2 --with=config-package
+	dh $@ --with systemd,python3 --with=config-package
 
 override_dh_auto_build:
 	make all

misc/qubes-desktop-run

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 
 from qubesagent.xdg import launch
 import sys

misc/qubes-session-autostart

@@ -1,9 +1,9 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
 # The Qubes OS Project, http://www.qubes-os.org
 #
-# Copyright (C) 2015  Marek Marczykowski-Górecki
+# Copyright (C) 2015-2019  Marek Marczykowski-Górecki
 #                                        <marmarek@invisiblethingslab.com>
 #
 # This program is free software; you can redistribute it and/or
@@ -21,7 +21,6 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 #
-import subprocess
 import sys
 
 from xdg.DesktopEntry import DesktopEntry
@@ -78,8 +77,9 @@ def process_autostart(environments):
                 if entry_should_be_started(entry, environments):
                     launch(entry_path, wait=False)
             except Exception as e:
-                print >>sys.stderr, "Failed to process '{}': {}".format(
+                print("Failed to process '{}': {}".format(
-                    entry_name, str(e)
+                    entry_name, str(e)),
+                    file=sys.stderr
                 )
 
 def main():

misc/qvm-features-request

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: fileencoding=utf-8
 
 #

qubes-rpc/qrun-in-vm

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # Send the command to the remote side, and then transfer stdin from local to
 # remote and stdout from remote to local.
 #

qubes-rpc/xdg-icon

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 import xdg.IconTheme
 import sys
 import os
@@ -7,15 +7,15 @@ themes = ['Humanity', 'Adwaita', 'gnome', 'oxygen']
 themes = themes + sorted([d for d in os.listdir("/usr/share/icons") if d not in themes and os.path.isdir("/usr/share/icons/" + d)])
 
 if len(sys.argv) < 3:
-	print "Usage:", sys.argv[0], "ICON SIZE"
+    print("Usage:", sys.argv[0], "ICON SIZE")
-	sys.exit(1)
+    sys.exit(1)
 
 for theme in themes:
-	icon = xdg.IconTheme.getIconPath(sys.argv[1], theme = theme, size = int(sys.argv[2]))
+    icon = xdg.IconTheme.getIconPath(sys.argv[1], theme = theme, size = int(sys.argv[2]))
-	if icon is not None:
+    if icon is not None:
-		break
+        break
 
 if icon is None:
-	sys.exit(1)
+    sys.exit(1)
 
-print icon
+print(icon)

qubesagent/firewall.py

@@ -1,4 +1,3 @@
-#!/usr/bin/python2 -O
 # vim: fileencoding=utf-8
 
 #
@@ -51,13 +50,13 @@ class FirewallWorker(object):
         self.log.addHandler(logging.StreamHandler(sys.stderr))
 
     def init(self):
-        '''Create appropriate chains/tables'''
+        """Create appropriate chains/tables"""
         raise NotImplementedError
 
     def sd_notify(self, state):
-        '''Send notification to systemd, if available'''
+        """Send notification to systemd, if available"""
         # based on sdnotify python module
-        if not 'NOTIFY_SOCKET' in os.environ:
+        if 'NOTIFY_SOCKET' not in os.environ:
             return
         addr = os.environ['NOTIFY_SOCKET']
         if addr[0] == '@':
@@ -71,39 +70,40 @@ class FirewallWorker(object):
             pass
 
     def cleanup(self):
-        '''Remove tables/chains - reverse work done by init'''
+        """Remove tables/chains - reverse work done by init"""
         raise NotImplementedError
 
     def apply_rules(self, source_addr, rules):
-        '''Apply rules in given source address'''
+        """Apply rules in given source address"""
         raise NotImplementedError
 
     def run_firewall_dir(self):
-        '''Run scripts dir contents, before user script'''
+        """Run scripts dir contents, before user script"""
         script_dir_paths = ['/etc/qubes/qubes-firewall.d',
-                      '/rw/config/qubes-firewall.d']
+                            '/rw/config/qubes-firewall.d']
         for script_dir_path in script_dir_paths:
-           if not os.path.isdir(script_dir_path):
+            if not os.path.isdir(script_dir_path):
-               continue
+                continue
-           for d_script in sorted(os.listdir(script_dir_path)):
+            for d_script in sorted(os.listdir(script_dir_path)):
-               d_script_path = os.path.join(script_dir_path, d_script)
+                d_script_path = os.path.join(script_dir_path, d_script)
-               if os.path.isfile(d_script_path) and \
+                if os.path.isfile(d_script_path) and \
-                       os.access(d_script_path, os.X_OK):
+                        os.access(d_script_path, os.X_OK):
-                   subprocess.call([d_script_path])
+                    subprocess.call([d_script_path])
 
     def run_user_script(self):
-        '''Run user script in /rw/config'''
+        """Run user script in /rw/config"""
         user_script_path = '/rw/config/qubes-firewall-user-script'
         if os.path.isfile(user_script_path) and \
                 os.access(user_script_path, os.X_OK):
             subprocess.call([user_script_path])
 
     def read_rules(self, target):
-        '''Read rules from QubesDB and return them as a list of dicts'''
+        """Read rules from QubesDB and return them as a list of dicts"""
         entries = self.qdb.multiread('/qubes-firewall/{}/'.format(target))
         assert isinstance(entries, dict)
         # drop full path
-        entries = dict(((k.split('/')[3], v) for k, v in entries.items()))
+        entries = dict(((k.split('/')[3], v.decode())
+                        for k, v in entries.items()))
         if 'policy' not in entries:
             raise RuleParseError('No \'policy\' defined')
         policy = entries.pop('policy')
@@ -196,7 +196,7 @@ class FirewallWorker(object):
 
 class IptablesWorker(FirewallWorker):
     supported_rule_opts = ['action', 'proto', 'dst4', 'dst6', 'dsthost',
-        'dstports', 'specialtarget', 'icmptype']
+                           'dstports', 'specialtarget', 'icmptype']
 
     def __init__(self):
         super(IptablesWorker, self).__init__()
@@ -207,7 +207,7 @@ class IptablesWorker(FirewallWorker):
 
     @staticmethod
     def chain_for_addr(addr):
-        '''Generate iptables chain name for given source address address'''
+        """Generate iptables chain name for given source address address"""
         return 'qbs-' + addr.replace('.', '-').replace(':', '-')[-20:]
 
     def run_ipt(self, family, args, **kwargs):
@@ -221,17 +221,17 @@ class IptablesWorker(FirewallWorker):
         # pylint: disable=no-self-use
         if family == 6:
             return subprocess.Popen(['ip6tables-restore'] + args,
-                stdin=subprocess.PIPE,
+                                    stdin=subprocess.PIPE,
-                stdout=subprocess.PIPE,
+                                    stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT)
+                                    stderr=subprocess.STDOUT)
         else:
             return subprocess.Popen(['iptables-restore'] + args,
-                stdin=subprocess.PIPE,
+                                    stdin=subprocess.PIPE,
-                stdout=subprocess.PIPE,
+                                    stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT)
+                                    stderr=subprocess.STDOUT)
 
     def create_chain(self, addr, chain, family):
-        '''
+        """
         Create iptables chain and hook traffic coming from `addr` to it.
 
         :param addr: source IP from which traffic should be handled by the
@@ -239,7 +239,7 @@ class IptablesWorker(FirewallWorker):
         :param chain: name of the chain to create
         :param family: address family (4 or 6)
         :return: None
-        '''
+        """
 
         self.run_ipt(family, ['-N', chain])
         self.run_ipt(family,
@@ -247,7 +247,7 @@ class IptablesWorker(FirewallWorker):
         self.chains[family].add(chain)
 
     def prepare_rules(self, chain, rules, family):
-        '''
+        """
         Helper function to translate rules list into input for iptables-restore
 
         :param chain: name of the chain to put rules into
@@ -255,7 +255,7 @@ class IptablesWorker(FirewallWorker):
         :param family: address family (4 or 6)
         :return: input for iptables-restore
         :rtype: str
-        '''
+        """
 
         iptables = "*filter\n"
 
@@ -359,7 +359,7 @@ class IptablesWorker(FirewallWorker):
         return iptables
 
     def apply_rules_family(self, source, rules, family):
-        '''
+        """
         Apply rules for given source address.
         Handle only rules for given address family (IPv4 or IPv6).
 
@@ -367,7 +367,7 @@ class IptablesWorker(FirewallWorker):
         :param rules: rules list
         :param family: address family, either 4 or 6
         :return: None
-        '''
+        """
 
         chain = self.chain_for_addr(source)
         if chain not in self.chains[family]:
@@ -377,7 +377,7 @@ class IptablesWorker(FirewallWorker):
         try:
             self.run_ipt(family, ['-F', chain])
             p = self.run_ipt_restore(family, ['-n'])
-            (output, _) = p.communicate(iptables)
+            (output, _) = p.communicate(iptables.encode())
             if p.returncode != 0:
                 raise RuleApplyError(
                     'iptables-restore failed: {}'.format(output))
@@ -417,7 +417,7 @@ class IptablesWorker(FirewallWorker):
 
 class NftablesWorker(FirewallWorker):
     supported_rule_opts = ['action', 'proto', 'dst4', 'dst6', 'dsthost',
-        'dstports', 'specialtarget', 'icmptype']
+                           'dstports', 'specialtarget', 'icmptype']
 
     def __init__(self):
         super(NftablesWorker, self).__init__()
@@ -428,21 +428,21 @@ class NftablesWorker(FirewallWorker):
 
     @staticmethod
     def chain_for_addr(addr):
-        '''Generate iptables chain name for given source address address'''
+        """Generate iptables chain name for given source address address"""
         return 'qbs-' + addr.replace('.', '-').replace(':', '-')
 
     def run_nft(self, nft_input):
         # pylint: disable=no-self-use
         p = subprocess.Popen(['nft', '-f', '/dev/stdin'],
-            stdin=subprocess.PIPE,
+                             stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT)
+                             stderr=subprocess.STDOUT)
-        stdout, _ = p.communicate(nft_input)
+        stdout, _ = p.communicate(nft_input.encode())
         if p.returncode != 0:
             raise RuleApplyError('nft failed: {}'.format(stdout))
 
     def create_chain(self, addr, chain, family):
-        '''
+        """
         Create iptables chain and hook traffic coming from `addr` to it.
 
         :param addr: source IP from which traffic should be handled by the
@@ -450,7 +450,7 @@ class NftablesWorker(FirewallWorker):
         :param chain: name of the chain to create
         :param family: address family (4 or 6)
         :return: None
-        '''
+        """
         nft_input = (
             'table {family} {table} {{\n'
             '  chain {chain} {{\n'
@@ -469,7 +469,7 @@ class NftablesWorker(FirewallWorker):
         self.chains[family].add(chain)
 
     def prepare_rules(self, chain, rules, family):
-        '''
+        """
         Helper function to translate rules list into input for iptables-restore
 
         :param chain: name of the chain to put rules into
@@ -477,7 +477,7 @@ class NftablesWorker(FirewallWorker):
         :param family: address family (4 or 6)
         :return: input for iptables-restore
         :rtype: str
-        '''
+        """
 
         assert family in (4, 6)
         nft_rules = []
@@ -517,7 +517,6 @@ class NftablesWorker(FirewallWorker):
                         else rule['proto']
                     nft_rule += ' ip6 nexthdr {}'.format(proto)
 
-
             if 'dst4' in rule:
                 nft_rule += ' ip daddr {}'.format(rule['dst4'])
             elif 'dst6' in rule:
@@ -587,7 +586,7 @@ class NftablesWorker(FirewallWorker):
             ))
 
     def apply_rules_family(self, source, rules, family):
-        '''
+        """
         Apply rules for given source address.
         Handle only rules for given address family (IPv4 or IPv6).
 
@@ -595,7 +594,7 @@ class NftablesWorker(FirewallWorker):
         :param rules: rules list
         :param family: address family, either 4 or 6
         :return: None
-        '''
+        """
 
         chain = self.chain_for_addr(source)
         if chain not in self.chains[family]:
@@ -649,5 +648,6 @@ def main():
     with context:
         worker.main()
 
+
 if __name__ == '__main__':
     main()

qubesagent/test_firewall.py

@@ -14,7 +14,7 @@ class DummyIptablesRestore(object):
         self.returncode = 0
 
     def communicate(self, stdin=None):
-        self._worker_mock.loaded_iptables[self._family] = stdin
+        self._worker_mock.loaded_iptables[self._family] = stdin.decode()
         return ("", None)
 
 class DummyQubesDB(object):
@@ -480,18 +480,18 @@ class TestFirewallWorker(TestCase):
         self.obj = FirewallWorker()
         rules = {
             '10.137.0.1': {
-                'policy': 'accept',
+                'policy': b'accept',
-                '0000': 'proto=tcp dstports=80-80 action=drop',
+                '0000': b'proto=tcp dstports=80-80 action=drop',
-                '0001': 'proto=udp specialtarget=dns action=accept',
+                '0001': b'proto=udp specialtarget=dns action=accept',
-                '0002': 'proto=udp action=drop',
+                '0002': b'proto=udp action=drop',
             },
-            '10.137.0.2': {'policy': 'accept'},
+            '10.137.0.2': {'policy': b'accept'},
             # no policy
-            '10.137.0.3': {'0000': 'proto=tcp action=accept'},
+            '10.137.0.3': {'0000': b'proto=tcp action=accept'},
             # no action
             '10.137.0.4': {
-                'policy': 'drop',
+                'policy': b'drop',
-                '0000': 'proto=tcp'
+                '0000': b'proto=tcp'
             },
         }
         for addr, entries in rules.items():

rpm_spec/core-agent.spec.in

@@ -124,12 +124,12 @@ Requires:   hostname
 Requires:   xterm
 # for qubes-desktop-run
 Requires:   pygobject3-base
-Requires:   dbus-python
+Requires:   python3-dbus
 # for qubes-session-autostart, xdg-icon
-Requires:   pyxdg
+Requires:   python3-pyxdg
-Requires:   python-daemon
+Requires:   python3-daemon
 # for qvm-feature-request
-Requires:   python2-qubesdb
+Requires:   python3-qubesdb
 Requires:   ImageMagick
 Requires:   librsvg2-tools
 Requires:   zenity
@@ -137,11 +137,8 @@ Requires:   dconf
 Requires:   qubes-core-qrexec-vm
 Requires:   qubes-libvchan
 Requires:   qubes-db-vm
-%if 0%{?fedora} >= 23
 Requires:   python%{python3_pkgversion}-dnf-plugins-qubes-hooks
-%else
+Requires: python%{python3_pkgversion}-setuptools
-Requires:   python2-dnf-plugins-qubes-hooks
-%endif
 Obsoletes:  qubes-core-vm-kernel-placeholder <= 1.0
 Obsoletes:  qubes-upgrade-vm < 3.2
 Provides:   qubes-core-vm = %{version}-%{release}
@@ -156,9 +153,7 @@ BuildRequires: libX11-devel
 BuildRequires: qubes-utils-devel >= 3.1.3
 BuildRequires: qubes-libvchan-@BACKEND_VMM@-devel
 BuildRequires: pam-devel
-%if 0%{?rhel} >= 7
+BuildRequires: python%{python3_pkgversion}-setuptools
-BuildRequires: python-setuptools
-%endif
 BuildRequires: systemd
 Source0: %{name}-%{version}.tar.gz
 
@@ -651,7 +646,6 @@ rm -f %{name}-%{version}
 /usr/lib/qubes/init/resize-rootfs-if-needed.sh
 /usr/lib/qubes/init/setup-rw.sh
 /usr/lib/qubes/init/setup-rwdev.sh
-/usr/lib/qubes/init/functions
 %dir /usr/lib/qubes-bind-dirs.d
 /usr/lib/qubes-bind-dirs.d/30_cron.conf
 /usr/share/applications/qubes-run-terminal.desktop
@@ -663,13 +657,14 @@ rm -f %{name}-%{version}
 /usr/share/glib-2.0/schemas/20_org.gnome.desktop.wm.preferences.qubes.gschema.override
 %{_mandir}/man1/qvm-*.1*
 
-%dir %{python2_sitelib}/qubesagent-*-py2.7.egg-info
+%dir %{python3_sitelib}/qubesagent-*-py*.egg-info
-%{python2_sitelib}/qubesagent-*-py2.7.egg-info/*
+%{python3_sitelib}/qubesagent-*-py*.egg-info/*
-%dir %{python2_sitelib}/qubesagent
+%{python3_sitelib}/qubesagent/__pycache__
-%{python2_sitelib}/qubesagent/__init__.py*
+%dir %{python3_sitelib}/qubesagent
-%{python2_sitelib}/qubesagent/firewall.py*
+%{python3_sitelib}/qubesagent/__init__.py*
-%{python2_sitelib}/qubesagent/test_firewall.py*
+%{python3_sitelib}/qubesagent/firewall.py*
-%{python2_sitelib}/qubesagent/xdg.py*
+%{python3_sitelib}/qubesagent/test_firewall.py*
+%{python3_sitelib}/qubesagent/xdg.py*
 
 /usr/share/qubes/mime-override/globs
 /usr/share/qubes/qubes-master-key.asc