Merge branch 'core3-devel'

.coveragerc

@@ -0,0 +1,3 @@
+[run]
+source = qubesagent
+omit = qubesagent/test*

.gitignore

@@ -4,3 +4,5 @@ deb/*
 *.pyo
 *~
 *.o
+.coverage
+*.egg-info

.travis.yml

@@ -1,6 +1,7 @@
 sudo: required
 dist: trusty
-language: generic
+language: python
+python: '3.5'
 install: git clone https://github.com/QubesOS/qubes-builder ~/qubes-builder
 script: ~/qubes-builder/scripts/travis-build
 env:
@@ -8,3 +9,22 @@ env:
  - DISTS_VM=fc24 USE_QUBES_REPO_VERSION=3.2 USE_QUBES_REPO_TESTING=1
  - DISTS_VM=jessie USE_QUBES_REPO_VERSION=3.2 USE_QUBES_REPO_TESTING=1
  - DISTS_VM=stretch USE_QUBES_REPO_VERSION=3.2 USE_QUBES_REPO_TESTING=1
+
+jobs:
+  include:
+    - python: '3.5'
+      install: pip install --quiet -r ci/requirements.txt
+      env: TESTS_ONLY=1
+      script:
+       - ./run-tests
+      after_success:
+       - codecov
+    - stage: deploy
+      python: '3.5'
+      env: DIST_DOM0=fc25 TESTS_ONLY=
+      script: ~/qubes-builder/scripts/travis-deploy
+
+
+branches:
+  except:
+    - /.*_.*/

Makefile

@@ -5,6 +5,7 @@ VERSION := $(shell cat version)
 DIST ?= fc18
 KDESERVICEDIR ?= /usr/share/kde4/services
 SBINDIR ?= /usr/sbin
+BINDIR ?= /usr/bin
 LIBDIR ?= /usr/lib
 SYSLIBDIR ?= /lib
 
@@ -132,7 +133,6 @@ install-sysvinit: install-init
 	install vm-init.d/qubes-core $(DESTDIR)/etc/init.d/
 	install vm-init.d/qubes-core-netvm $(DESTDIR)/etc/init.d/
 	install vm-init.d/qubes-firewall $(DESTDIR)/etc/init.d/
-	install vm-init.d/qubes-netwatcher $(DESTDIR)/etc/init.d/
 	install vm-init.d/qubes-qrexec-agent $(DESTDIR)/etc/init.d/
 	install vm-init.d/qubes-updates-proxy $(DESTDIR)/etc/init.d/
 	install vm-init.d/qubes-dvm $(DESTDIR)/etc/init.d/
@@ -142,10 +142,6 @@ install-sysvinit: install-init
 
 install-rh: install-systemd install-systemd-dropins install-sysvinit
 	install -D -m 0644 misc/qubes-r3.repo $(DESTDIR)/etc/yum.repos.d/qubes-r3.repo
-	install -d $(DESTDIR)/usr/share/glib-2.0/schemas/
-	install -m 0644 misc/org.gnome.settings-daemon.plugins.updates.gschema.override $(DESTDIR)/usr/share/glib-2.0/schemas/
-	install -m 0644 misc/org.gnome.nautilus.gschema.override $(DESTDIR)/usr/share/glib-2.0/schemas/
-	install -m 0644 misc/org.mate.NotificationDaemon.gschema.override $(DESTDIR)/usr/share/glib-2.0/schemas/
 	install -d $(DESTDIR)$(LIBDIR)/yum-plugins/
 	install -m 0644 misc/yum-qubes-hooks.py* $(DESTDIR)$(LIBDIR)/yum-plugins/
 	install -D -m 0644 misc/yum-qubes-hooks.conf $(DESTDIR)/etc/yum/pluginconf.d/yum-qubes-hooks.conf
@@ -174,6 +170,11 @@ install-common:
 	$(MAKE) -C autostart-dropins install
 	install -m 0644 -D misc/fstab $(DESTDIR)/etc/fstab
 
+	# force /usr/bin before /bin to have /usr/bin/python instead of /bin/python
+	PATH="/usr/bin:$(PATH)" python3 setup.py install -O1 --root $(DESTDIR)
+	mkdir -p $(DESTDIR)$(SBINDIR)
+	mv $(DESTDIR)/usr/bin/qubes-firewall $(DESTDIR)$(SBINDIR)/qubes-firewall
+
 	install -d -m 0750 $(DESTDIR)/etc/sudoers.d/
 	install -D -m 0440 misc/qubes.sudoers $(DESTDIR)/etc/sudoers.d/qubes
 	install -D -m 0440 misc/sudoers.d_qt_x11_no_mitshm $(DESTDIR)/etc/sudoers.d/qt_x11_no_mitshm
@@ -181,7 +182,7 @@ install-common:
 
 	install -d $(DESTDIR)/var/lib/qubes
 
-	install -D misc/xenstore-watch $(DESTDIR)/usr/bin/xenstore-watch-qubes
+	install -D misc/xenstore-watch $(DESTDIR)$(BINDIR)/xenstore-watch-qubes
 	install -d $(DESTDIR)/etc/udev/rules.d
 	install -m 0644 misc/udev-qubes-misc.rules $(DESTDIR)/etc/udev/rules.d/50-qubes-misc.rules
 	install -d $(DESTDIR)$(LIBDIR)/qubes/
@@ -192,6 +193,12 @@ install-common:
 	install -D -m 0644 misc/polkit-1-qubes-allow-all.rules $(DESTDIR)/etc/polkit-1/rules.d/00-qubes-allow-all.rules
 	install -D -m 0644 misc/mime-globs $(DESTDIR)/usr/share/qubes/mime-override/globs
 	install misc/qubes-download-dom0-updates.sh $(DESTDIR)$(LIBDIR)/qubes/
+	install -d $(DESTDIR)/usr/share/glib-2.0/schemas/
+	install -m 0644 \
+		misc/20_org.gnome.settings-daemon.plugins.updates.qubes.gschema.override \
+		misc/20_org.gnome.nautilus.qubes.gschema.override \
+		misc/20_org.mate.NotificationDaemon.qubes.gschema.override \
+		$(DESTDIR)/usr/share/glib-2.0/schemas/
 	install -g user -m 2775 -d $(DESTDIR)/var/lib/qubes/dom0-updates
 	install -D -m 0644 misc/qubes-master-key.asc $(DESTDIR)/usr/share/qubes/qubes-master-key.asc
 
@@ -213,6 +220,7 @@ install-common:
 	install -d $(DESTDIR)/usr/lib/NetworkManager/conf.d
 	install -m 0644 network/nm-30-qubes.conf $(DESTDIR)/usr/lib/NetworkManager/conf.d/30-qubes.conf
 	install -D network/vif-route-qubes $(DESTDIR)/etc/xen/scripts/vif-route-qubes
+	install -D network/vif-qubes-nat.sh $(DESTDIR)/etc/xen/scripts/vif-qubes-nat.sh
 	install -m 0644 -D network/tinyproxy-updates.conf $(DESTDIR)/etc/tinyproxy/tinyproxy-updates.conf
 	install -m 0644 -D network/updates-blacklist $(DESTDIR)/etc/tinyproxy/updates-blacklist
 	install -m 0755 -D network/iptables-updates-proxy $(DESTDIR)$(LIBDIR)/qubes/iptables-updates-proxy
@@ -223,14 +231,9 @@ install-common:
 	install -m 0400 -D network/ip6tables $(DESTDIR)/etc/qubes/ip6tables.rules
 	install -m 0755 network/update-proxy-configs $(DESTDIR)$(LIBDIR)/qubes/
 
-
-	install -d $(DESTDIR)/$(SBINDIR)
-	install network/qubes-firewall $(DESTDIR)/$(SBINDIR)/
-	install network/qubes-netwatcher $(DESTDIR)/$(SBINDIR)/
-
-	install -d $(DESTDIR)/usr/bin
-	install -m 0755 misc/qubes-session-autostart $(DESTDIR)/usr/bin/qubes-session-autostart
-
+	install -d $(DESTDIR)$(BINDIR)
+	install -m 0755 misc/qubes-session-autostart $(DESTDIR)$(BINDIR)/qubes-session-autostart
+	install -m 0755 misc/qvm-features-request $(DESTDIR)$(BINDIR)/qvm-features-request
 	install qubes-rpc/{qvm-open-in-dvm,qvm-open-in-vm,qvm-copy-to-vm,qvm-run,qvm-mru-entry} $(DESTDIR)/usr/bin
 	ln -s qvm-copy-to-vm $(DESTDIR)/usr/bin/qvm-move-to-vm
 	install qubes-rpc/qvm-copy-to-vm.kde $(DESTDIR)$(LIBDIR)/qubes
@@ -239,7 +242,7 @@ install-common:
 	install qubes-rpc/qvm-move-to-vm.gnome $(DESTDIR)$(LIBDIR)/qubes
 	install qubes-rpc/xdg-icon $(DESTDIR)$(LIBDIR)/qubes
 	install qubes-rpc/{vm-file-editor,qfile-agent,qopen-in-vm} $(DESTDIR)$(LIBDIR)/qubes
-	install qubes-rpc/qubes-open $(DESTDIR)/usr/bin
+	install qubes-rpc/qubes-open $(DESTDIR)$(BINDIR)
 	install qubes-rpc/tar2qfile $(DESTDIR)$(LIBDIR)/qubes
 	# Install qfile-unpacker as SUID - because it will fail to receive files from other vm
 	install -m 4755  qubes-rpc/qfile-unpacker $(DESTDIR)$(LIBDIR)/qubes
@@ -262,6 +265,8 @@ install-common:
 	install -m 0644 qubes-rpc/qubes.GetImageRGBA $(DESTDIR)/etc/qubes-rpc
 	install -m 0644 qubes-rpc/qubes.SetDateTime $(DESTDIR)/etc/qubes-rpc
 	install -m 0755 qubes-rpc/qubes.InstallUpdatesGUI $(DESTDIR)/etc/qubes-rpc
+	install -m 0755 qubes-rpc/qubes.ResizeDisk $(DESTDIR)/etc/qubes-rpc
+	install -m 0755 qubes-rpc/qubes.StartApp $(DESTDIR)/etc/qubes-rpc
 
 	install -d $(DESTDIR)/etc/qubes/suspend-pre.d
 	install -m 0644 qubes-rpc/suspend-pre.README $(DESTDIR)/etc/qubes/suspend-pre.d/README
@@ -271,14 +276,14 @@ install-common:
 	install -d $(DESTDIR)/usr/share/nautilus-python/extensions
 	install -m 0644 qubes-rpc/*_nautilus.py $(DESTDIR)/usr/share/nautilus-python/extensions
 
-	install -D -m 0755 misc/qubes-desktop-run $(DESTDIR)/usr/bin/qubes-desktop-run
+	install -D -m 0755 misc/qubes-desktop-run $(DESTDIR)$(BINDIR)/qubes-desktop-run
 
 	mkdir -p $(DESTDIR)/$(PYTHON_SITEARCH)/qubes/
 
 ifeq ($(shell lsb_release -is), Debian)
-	install -m 0644 misc/xdg.py $(DESTDIR)/$(PYTHON_SITEARCH)/qubes/
+	install -m 0644 misc/qubesxdg.py $(DESTDIR)/$(PYTHON2_SITELIB)/
 else
-	install -m 0644 misc/py2/xdg.py* $(DESTDIR)/$(PYTHON_SITEARCH)/qubes/
+	install -m 0644 misc/py2/qubesxdg.py* $(DESTDIR)/$(PYTHON2_SITELIB)/
 endif
 
 ifneq (,$(filter xenial stretch, $(shell lsb_release -cs)))
@@ -311,8 +316,6 @@ install-deb: install-common install-systemd install-systemd-dropins
 	install -m 0644 misc/pam.d_su.qubes $(DESTDIR)/etc/pam.d/su.qubes
 	install -d $(DESTDIR)/etc/needrestart/conf.d
 	install -D -m 0644 misc/50_qubes.conf $(DESTDIR)/etc/needrestart/conf.d/50_qubes.conf
-	install -d $(DESTDIR)/usr/share/glib-2.0/schemas/
-	install -m 0644 misc/org.gnome.nautilus.gschema.override $(DESTDIR)/usr/share/glib-2.0/schemas/
 
 
 install-vm: install-rh install-common

archlinux/PKGBUILD

@@ -56,7 +56,6 @@ sed 's:#!/usr/bin/env python:#!/usr/bin/env python2:' -i qubes-rpc/*
 
 # Fix for archlinux sbindir
 sed 's:/usr/sbin/ntpdate:/usr/bin/ntpdate:g' -i qubes-rpc/sync-ntp-clock
-sed 's:/usr/sbin/qubes-netwatcher:/usr/bin/qubes-netwatcher:g' -i vm-systemd/qubes-netwatcher.service
 sed 's:/usr/sbin/qubes-firewall:/usr/bin/qubes-firewall:g' -i vm-systemd/qubes-firewall.service
 
 for dir in qubes-rpc qrexec misc; do

archlinux/PKGBUILD.install

@@ -475,7 +475,7 @@ post_remove() {
 
     rm -rf /var/lib/qubes/xdg
 
-    for srv in qubes-dvm qubes-sysinit qubes-misc-post qubes-mount-dirs qubes-netwatcher qubes-network qubes-qrexec-agent; do
+    for srv in qubes-dvm qubes-sysinit qubes-misc-post qubes-mount-dirs qubes-network qubes-qrexec-agent; do
         systemctl disable $srv.service
     done
     

ci/requirements.txt

@@ -0,0 +1,6 @@
+# WARNING: those requirements are used only for travis-ci.org
+# they SHOULD NOT be used under normal conditions; use system package manager
+docutils
+pylint
+codecov
+python-daemon

debian/control

@@ -7,6 +7,7 @@ Build-Depends:
     libqubes-rpc-filecopy-dev (>= 3.1.3),
     libvchan-xen-dev,
     python,
+    python3-setuptools,
     debhelper,
     quilt,
     libxen-dev,
@@ -40,6 +41,7 @@ Depends:
     procps,
     util-linux,
     python2.7,
+    python-daemon,
     python-gi,
     python-xdg,
     python-dbus,

debian/qubes-core-agent.postrm

@@ -43,7 +43,7 @@ if [ "${1}" = "remove" ] ; then
         rm /lib/firmware/updates
     fi
 
-    for srv in qubes-dvm qubes-sysinit qubes-misc-post qubes-netwatcher qubes-network qubes-qrexec-agent; do
+    for srv in qubes-dvm qubes-sysinit qubes-misc-post qubes-network qubes-qrexec-agent; do
         systemctl disable ${srv}.service
     done
 fi

misc/Makefile

@@ -23,7 +23,7 @@ python2:
 python3:
 	rm -rf py3
 	mkdir -p py3
-	cp dnf-qubes-hooks.py xdg.py py3/
+	cp dnf-qubes-hooks.py qubesxdg.py py3/
 	python3 -m compileall py3
 	python3 -O -m compileall py3
 

misc/qubes-desktop-run

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 
-from qubes.xdg import launch
+from qubesxdg import launch
 import sys
 
 if __name__ == '__main__':

misc/qubes-download-dom0-updates.sh

@@ -2,7 +2,6 @@
 
 DOM0_UPDATES_DIR=/var/lib/qubes/dom0-updates
 
-DOIT=0
 GUI=1
 CLEAN=0
 CHECK_ONLY=0
@@ -17,7 +16,7 @@ export LC_ALL=C
 while [ -n "$1" ]; do
     case "$1" in
         --doit)
-            DOIT=1
+            # ignore
             ;;
         --nogui)
             GUI=0
@@ -80,48 +79,30 @@ if [ "$CLEAN" = "1" ]; then
     rm -rf $DOM0_UPDATES_DIR/var/cache/yum/*
 fi
 
-if [ "x$PKGLIST" = "x" ]; then
+# just check for updates, but don't download any package
+if [ "x$PKGLIST" = "x" -a "$CHECK_ONLY" = "1" ]; then
     echo "Checking for dom0 updates..." >&2
     UPDATES_FULL=`$YUM $OPTS check-update`
     check_update_retcode=$?
-    UPDATES_FULL=`echo "$UPDATES_FULL" | grep -v "^Loaded plugins:\|^Last metadata\|^$"`
     if [ $check_update_retcode -eq 1 ]; then
         # Exit here if yum have reported an error. Exit code 100 isn't an
         # error, it's "updates available" info, so check specifically for exit code 1
         exit 1
     fi
-    UPDATES=`echo "$UPDATES_FULL" | grep -v "^Obsoleting\|Could not" | cut -f 1 -d ' '`
-    if [ -z "$UPDATES" -a $check_update_retcode -eq 100 ]; then
-        # save not empty string for below condition (-z "$UPDATES"), but blank
-        # to not confuse the user wwith magic strings in messages
-        UPDATES=" "
-    elif [ $check_update_retcode -eq 0 ]; then
-        # exit code 0 means no updates available - regardless of stdout messages
-        UPDATES=""
-    fi
-else
-    PKGS_FROM_CMDLINE=1
-fi
-
-if [ -z "$PKGLIST" -a -z "$UPDATES" ]; then
-    echo "No new updates available"
-    if [ "$GUI" = 1 ]; then
-        zenity --info --text="No new updates available"
+    if [ $check_update_retcode -eq 100 ]; then
+        echo "Available updates: "
+        echo "$UPDATES_FULL"
+        exit 100
+    else
+        echo "No new updates available"
+        if [ "$GUI" = 1 ]; then
+            zenity --info --text="No new updates available"
+        fi
+        exit 0
     fi
-    exit 0
-fi
-
-if [ "$CHECK_ONLY" = "1" ]; then
-    echo "Available updates: "
-    echo "$UPDATES_FULL"
-    exit 100
-fi
-
-if [ "$DOIT" != "1" -a "$PKGS_FROM_CMDLINE" != "1" ]; then
-    zenity --question --title="Qubes Dom0 updates" \
-      --text="There are updates for dom0 available, do you want to download them now?" || exit 0
 fi
 
+# now, we will download something
 YUM_COMMAND="fakeroot $YUM $YUM_ACTION -y --downloadonly"
 # check for --downloadonly option - if not supported (Debian), fallback to
 # yumdownloader
@@ -135,6 +116,15 @@ if ! $YUM --help | grep -q downloadonly; then
         exit 1
     fi
     if [ "$YUM_ACTION" = "upgrade" ]; then
+        UPDATES_FULL=`$YUM $OPTS check-update $PKGLIST`
+        check_update_retcode=$?
+        UPDATES_FULL=`echo "$UPDATES_FULL" | grep -v "^Loaded plugins:\|^Last metadata\|^$"`
+        UPDATES=`echo "$UPDATES_FULL" | grep -v "^Obsoleting\|Could not" | cut -f 1 -d ' '`
+        if [ $check_update_retcode -eq 0 ]; then
+            # exit code 0 means no updates available - regardless of stdout messages
+            echo "No new updates available"
+            exit 0
+        fi
         PKGLIST=$UPDATES
     fi
     YUM_COMMAND="yumdownloader --destdir=$DOM0_UPDATES_DIR/packages --resolve"

misc/qubes-session-autostart

@@ -25,7 +25,7 @@ import subprocess
 import sys
 
 from xdg.DesktopEntry import DesktopEntry
-from qubes.xdg import launch
+from qubesxdg import launch
 import xdg.BaseDirectory
 import os
 
@@ -86,4 +86,4 @@ def main():
     process_autostart(sys.argv[1:])
 
 if __name__ == '__main__':
-    main()
+    main()

misc/qvm-features-request

@@ -0,0 +1,81 @@
+#!/usr/bin/env python2
+# vim: fileencoding=utf-8
+
+#
+# The Qubes OS Project, https://www.qubes-os.org/
+#
+# Copyright (C) 2010-2016  Joanna Rutkowska <joanna@invisiblethingslab.com>
+# Copyright (C)      2016  Wojtek Porczyk <woju@invisiblethingslab.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+import argparse
+import os
+import subprocess
+import sys
+
+import qubesdb
+
+class FeatureRequestAction(argparse.Action):
+    '''Action for argument parser that stores a property.'''
+    # pylint: disable=redefined-builtin,too-few-public-methods
+    def __init__(self,
+            option_strings,
+            dest='features',
+            metavar='NAME=VALUE',
+            required=False,
+            help='request a feature with the value'):
+        super(FeatureRequestAction, self).__init__(option_strings, dest=dest,
+            metavar=metavar, nargs='*', required=required, default={},
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        for request in values:
+            try:
+                feature, value = request.split('=', 1)
+            except ValueError:
+                parser.error(
+                    'invalid feature request token: {!r}'.format(request))
+
+            getattr(namespace, self.dest)[feature] = value
+
+
+parser = argparse.ArgumentParser(
+    description='submit a feature request to the dom0')
+
+parser.add_argument('--commit',
+    action='store_true', default=False,
+    help='actually send the request (without it, only make entries in qubesdb)')
+
+parser.add_argument('features',
+    action=FeatureRequestAction)
+
+
+def main(args=None):
+    args = parser.parse_args(args)
+
+    qdb = qubesdb.QubesDB()
+    for feature, value in args.features.items():
+        qdb.write('/features-request/' + feature, value)
+
+    if args.commit:
+        devnull = os.open(os.devnull, os.O_RDWR)
+        subprocess.check_call(
+            ['qrexec-client-vm', 'dom0', 'qubes.FeaturesRequest'],
+            stdin=devnull, stdout=devnull)
+
+if __name__ == '__main__':
+    sys.exit(main())

network/ip6tables

@@ -3,6 +3,8 @@
 :INPUT DROP [1:72]
 :FORWARD DROP [0:0]
 :OUTPUT ACCEPT [0:0]
+:QBS-FORWARD - [0:0]
 -A INPUT -i lo -j ACCEPT
+-A FORWARD -j QBS-FORWARD
 COMMIT
 # Completed on Tue Sep 25 16:00:20 2012

network/iptables

@@ -17,6 +17,7 @@ COMMIT
 :INPUT ACCEPT [168:11399]
 :FORWARD ACCEPT [0:0]
 :OUTPUT ACCEPT [128:12536]
+:QBS-FORWARD - [0:0]
 -A INPUT -i vif+ -p udp -m udp --dport 68 -j DROP
 -A INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
 -A INPUT -i vif+ -p icmp -j ACCEPT
@@ -24,6 +25,7 @@ COMMIT
 -A INPUT -i vif+ -j REJECT --reject-with icmp-host-prohibited
 -A INPUT -j DROP
 -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A FORWARD -j QBS-FORWARD
 -A FORWARD -i vif+ -o vif+ -j DROP
 -A FORWARD -i vif+ -j ACCEPT
 -A FORWARD -j DROP

network/qubes-firewall

@@ -1,62 +0,0 @@
-#!/bin/sh
-set -e
-
-PIDFILE=/var/run/qubes/qubes-firewall.pid
-XENSTORE_IPTABLES=/qubes-iptables
-XENSTORE_IPTABLES_HEADER=/qubes-iptables-header
-XENSTORE_ERROR=/qubes-iptables-error
-OLD_RULES=""
-# PIDfile handling
-[ -e "$PIDFILE" ] && kill -s 0 $(cat "$PIDFILE") 2>/dev/null && exit 0
-echo $$ >$PIDFILE
-
-trap 'exit 0' TERM
-
-FIRST_TIME=yes
-
-while true; do
-
-	echo "1" > /proc/sys/net/ipv4/ip_forward
-
-	if [ "$FIRST_TIME" ]; then
-		FIRST_TIME=
-		TRIGGER=reload
-	else
-		# Wait for changes in qubesdb file
-		qubesdb-watch $XENSTORE_IPTABLES
-		TRIGGER=$(qubesdb-read $XENSTORE_IPTABLES)
-	fi
-
-	if ! [ "$TRIGGER" = "reload" ]; then continue ; fi
-
-	# Disable forwarding to prevent potential "leaks" that might
-	# be bypassing the firewall or some proxy service (e.g. tor)
-	# during the time when the rules are being (re)applied
-	echo "0" > /proc/sys/net/ipv4/ip_forward
-
-	RULES=$(qubesdb-read $XENSTORE_IPTABLES_HEADER)
-	IPTABLES_SAVE=$(iptables-save | sed '/^\*filter/,/^COMMIT/d')
-	OUT=$(printf '%s\n%s\n' "$RULES" "$IPTABLES_SAVE" | sed 's/\\n\|\\x0a/\n/g' | iptables-restore 2>&1 || true)
-
-	for i in $(qubesdb-list -f /qubes-iptables-domainrules) ; do
-		RULES=$(qubesdb-read "$i")
-		ERRS=$(printf '%s\n' "$RULES" | sed 's/\\n\|\\x0a/\n/g' | /sbin/iptables-restore -n 2>&1 || true)
-		if [ -n "$ERRS" ]; then
-			echo "Failed applying rules for $i: $ERRS" >&2
-			OUT="$OUT$ERRS"
-		fi
-	done
-	qubesdb-write $XENSTORE_ERROR "$OUT"
-	if [ -n "$OUT" ]; then
-		DISPLAY=:0 /usr/bin/notify-send -t 3000 "Firewall loading error ($(hostname))" "$OUT" || :
-	fi
-
-	if [ `systemctl is-active qubes-updates-proxy` = "active" ]; then
-		iptables -I INPUT -i vif+ -p tcp --dport 8082 -j ACCEPT
-	fi
-
-	# Check if user didn't define some custom rules to be applied as well...
-	[ -x /rw/config/qubes-firewall-user-script ] && /rw/config/qubes-firewall-user-script
-	# XXX: Backward compatibility
-	[ -x /rw/config/qubes_firewall_user_script ] && /rw/config/qubes_firewall_user_script
-done

network/qubes-netwatcher

@@ -1,31 +0,0 @@
-#!/bin/sh
-set -e
-
-PIDFILE=/var/run/qubes/qubes-netwatcher.pid
-CURR_NETCFG=""
-
-# PIDfile handling
-[ -e "$PIDFILE" ] && kill -s 0 $(cat "$PIDFILE") 2>/dev/null && exit 0
-echo $$ >$PIDFILE
-
-trap 'exit 0' TERM
-
-while true; do
-	NET_DOMID=$(xenstore-read qubes-netvm-domid || :)
-	if [ -n "$NET_DOMID" ] && [ $NET_DOMID -gt 0 ]; then
-		UNTRUSTED_NETCFG=$(xenstore-read /local/domain/$NET_DOMID/qubes-netvm-external-ip || :)
-		# UNTRUSTED_NETCFG is not parsed in any way
-		# thus, no sanitization ready
-		# but be careful when passing it to other shell scripts
-		if [ "$UNTRUSTED_NETCFG" != "$CURR_NETCFG" ]; then
-			/sbin/service qubes-firewall stop
-			/sbin/service qubes-firewall start
-			CURR_NETCFG="$UNTRUSTED_NETCFG"
-			xenstore-write qubes-netvm-external-ip "$CURR_NETCFG"
-		fi
-
-		xenstore-watch -n 3 /local/domain/$NET_DOMID/qubes-netvm-external-ip qubes-netvm-domid
-	else
-		xenstore-watch -n 2 qubes-netvm-domid
-	fi
-done

network/vif-qubes-nat.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+#set -x
+
+undetectable_netvm_ips=
+
+netns="${vif}-nat"
+netvm_if="${vif}"
+netns_netvm_if="${vif}-p"
+netns_appvm_if="${vif}"
+
+#
+#               .----------------------------------.
+#               |          NetVM/ProxyVM           |
+# .------------.|.------------------.              |
+# |   AppVM    ||| $netns namespace |              |
+# |            |||                  |              |
+# |  eth0<--------->$netns_appvm_if |              |
+# |$appvm_ip   |||   $appvm_gw_ip   |              |
+# |$appvm_gw_ip|||         ^        |              |
+# '------------'||         |NAT     |              |
+#               ||         v        |              |
+#               ||  $netns_netvm_if<--->$netvm_if  |
+#               ||     $netvm_ip    |  $netvm_gw_ip|
+#               |'------------------'              |
+#               '----------------------------------'
+#
+
+
+function run
+{
+    #echo "$@" >> /var/log/qubes-nat.log
+    "$@"
+}
+
+function netns
+{
+    run ip netns exec "$netns" "$@"
+}
+
+run ip addr flush dev "$netns_appvm_if"
+run ip netns delete "$netns" || :
+
+if test "$command" == online; then
+    run ip netns add "$netns"
+    run ip link set "$netns_appvm_if" netns "$netns"
+
+    # keep the same MAC as the real vif interface, so NetworkManager will still
+    # ignore it
+    run ip link add "$netns_netvm_if" type veth peer name "$netvm_if" address fe:ff:ff:ff:ff:ff
+    run ip link set "$netns_netvm_if" netns "$netns"
+
+    netns ip6tables -t raw -I PREROUTING -j DROP
+    netns ip6tables -P INPUT DROP
+    netns ip6tables -P FORWARD DROP
+    netns ip6tables -P OUTPUT DROP
+
+    netns sh -c 'echo 1 > /proc/sys/net/ipv4/ip_forward'
+
+    netns iptables -t raw -I PREROUTING -i "$netns_appvm_if" ! -s "$appvm_ip" -j DROP
+
+    if test -n "$undetectable_netvm_ips"; then
+        # prevent an AppVM connecting to its own ProxyVM IP because that makes the internal IPs detectable even with no firewall rules
+        netns iptables -t raw -I PREROUTING -i "$netns_appvm_if" -d "$netvm_ip" -j DROP
+
+        # same for the gateway/DNS IPs
+        netns iptables -t raw -I PREROUTING -i "$netns_appvm_if" -d "$netvm_gw_ip" -j DROP
+        netns iptables -t raw -I PREROUTING -i "$netns_appvm_if" -d "$netvm_dns1_ip" -j DROP
+        netns iptables -t raw -I PREROUTING -i "$netns_appvm_if" -d "$netvm_dns2_ip" -j DROP
+    fi
+
+    netns iptables -t nat -I PREROUTING -i "$netns_netvm_if" -j DNAT --to-destination "$appvm_ip"
+    netns iptables -t nat -I POSTROUTING -o "$netns_netvm_if" -j SNAT --to-source "$netvm_ip"
+
+    netns iptables -t nat -I PREROUTING -i "$netns_appvm_if" -d "$appvm_gw_ip" -j DNAT --to-destination "$netvm_gw_ip"
+    netns iptables -t nat -I POSTROUTING -o "$netns_appvm_if" -s "$netvm_gw_ip" -j SNAT --to-source "$appvm_gw_ip"
+
+    if test -n "$appvm_dns1_ip"; then
+        netns iptables -t nat -I PREROUTING -i "$netns_appvm_if" -d "$appvm_dns1_ip" -j DNAT --to-destination "$netvm_dns1_ip"
+        netns iptables -t nat -I POSTROUTING -o "$netns_appvm_if" -s "$netvm_dns1_ip" -j SNAT --to-source "$appvm_dns1_ip"
+    fi
+
+    if test -n "$appvm_dns2_ip"; then
+        netns iptables -t nat -I PREROUTING -i "$netns_appvm_if" -d "$appvm_dns2_ip" -j DNAT --to-destination "$netvm_dns2_ip"
+        netns iptables -t nat -I POSTROUTING -o "$netns_appvm_if" -s "$netvm_dns2_ip" -j SNAT --to-source "$appvm_dns2_ip"
+    fi
+
+    netns ip addr add "$netvm_ip" dev "$netns_netvm_if"
+    netns ip addr add "$appvm_gw_ip" dev "$netns_appvm_if"
+
+    netns ip link set "$netns_netvm_if" up
+    netns ip link set "$netns_appvm_if" up
+
+    netns ip route add "$appvm_ip" dev "$netns_appvm_if" src "$appvm_gw_ip"
+    netns ip route add "$netvm_gw_ip" dev "$netns_netvm_if" src "$netvm_ip"
+    netns ip route add default via "$netvm_gw_ip" dev "$netns_netvm_if" src "$netvm_ip"
+
+
+    #run ip addr add "$netvm_gw_ip" dev "$netvm_if"
+    #run ip link set "$netvm_if" up
+    #run ip route add "$netvm_ip" dev "$netvm_if" src "$netvm_gw_ip"
+fi
+

network/vif-route-qubes

@@ -26,6 +26,31 @@ dir=$(dirname "$0")
 #main_ip=$(dom0_ip)
 lockfile=/var/run/xen-hotplug/vif-lock
 
+if [ "${ip}" ]; then
+    # IPs as seen by this VM
+    netvm_ip="$ip"
+    netvm_gw_ip=`qubesdb-read /qubes-netvm-gateway`
+    netvm_dns1_ip=`qubesdb-read /qubes-netvm-primary-dns`
+    netvm_dns2_ip=`qubesdb-read /qubes-netvm-secondary-dns`
+
+    back_ip="$netvm_gw_ip"
+
+    # IPs as seen by the VM - if other than $netvm_ip
+    appvm_gw_ip="`qubesdb-read /mapped-ip/$ip/visible-gateway 2>/dev/null || :`"
+    appvm_ip="`qubesdb-read /mapped-ip/$ip/visible-ip 2>/dev/null || :`"
+fi
+
+# Apply NAT if IP visible from the VM is different than the "real" one
+# See vif-qubes-nat.sh for details
+if [ -n "$appvm_ip" -a -n "$appvm_gw_ip" -a "$appvm_ip" != "$netvm_ip" ]; then
+    if test "$command" == online; then
+        echo 1 >/proc/sys/net/ipv4/conf/${vif}/proxy_arp
+    fi
+
+    . "$dir/vif-qubes-nat.sh"
+fi
+
+
 case "$command" in
 	online)
 		ifconfig ${vif} up
@@ -56,7 +81,6 @@ if [ "${ip}" ] ; then
 	done
 	echo -e "*raw\n$iptables_cmd -i ${vif} ! -s ${ip} -j DROP\nCOMMIT" | \
 		${cmdprefix} flock $lockfile iptables-restore --noflush
-	back_ip=`qubesdb-read /qubes-netvm-gateway`
 	${cmdprefix} ip addr ${ipcmd} ${back_ip}/32 dev ${vif}
 fi
 

qubes-rpc/qubes.ResizeDisk

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+read disk_name
+
+set -e
+
+case $disk_name in
+    private)
+        # force some read to refresh device size
+        head /dev/xvdb > /dev/null
+        resize2fs /dev/xvdb
+        ;;
+    root)
+        # force some read to refresh device size
+        head /dev/xvda > /dev/null
+        new_size=$(cat /sys/block/xvda/size)
+        ro=$(/sys/block/xvda/ro)
+        if [ $ro -eq 1 ]; then
+            new_table="0 $new_size snapshot /dev/xvda /dev/xvdc2 N 16"
+        else
+            new_table="0 $new_size linear /dev/xvda 0"
+        fi
+        dmsetup load dmroot --table "$new_table"
+        dmsetup resume dmroot
+        resize2fs /dev/mapper/dmroot
+        ;;
+    *)
+        echo "Automatic resize of '$disk_name' not supported" >&2
+        exit 1
+        ;;
+esac
+        

qubes-rpc/qubes.StartApp

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+if [ -z "$1" ]; then
+    echo "This service require an argument" >&2
+    exit 1
+fi
+
+# make sure it have .desktop suffix, and only one of it
+app_basename="${1%.desktop}.desktop"
+
+# Based on XDG Base Directory Specification, Version 0.7
+[ -n "$XDG_DATA_HOME" ] || XDG_DATA_HOME="$HOME/.local/share"
+[ -n "$XDG_DATA_DIRS" ] || XDG_DATA_DIRS="/usr/local/share:/usr/share"
+
+for dir in $(echo "$XDG_DATA_HOME:$XDG_DATA_DIRS" | tr : ' '); do
+    if ! [ -d "$dir/applications" ]; then
+        continue
+    fi
+    if [ -f "$dir/applications/$app_basename" ]; then
+        exec qubes-desktop-run "$dir/applications/$app_basename"
+    fi
+done
+echo "applications/$app_basename not found in $XDG_DATA_HOME:$XDG_DATA_DIRS" >&2
+exit 1

qubesagent/firewall.py

@@ -0,0 +1,576 @@
+#!/usr/bin/python2 -O
+# vim: fileencoding=utf-8
+
+#
+# The Qubes OS Project, https://www.qubes-os.org/
+#
+# Copyright (C) 2016
+#                   Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+import logging
+import os
+import socket
+import subprocess
+from distutils import spawn
+
+import daemon
+
+import qubesdb
+import sys
+
+import signal
+
+
+class RuleParseError(Exception):
+    pass
+
+
+class RuleApplyError(Exception):
+    pass
+
+
+class FirewallWorker(object):
+    def __init__(self):
+        self.terminate_requested = False
+        self.qdb = qubesdb.QubesDB()
+        self.log = logging.getLogger('qubes.firewall')
+        self.log.addHandler(logging.StreamHandler(sys.stderr))
+
+    def init(self):
+        '''Create appropriate chains/tables'''
+        raise NotImplementedError
+
+    def cleanup(self):
+        '''Remove tables/chains - reverse work done by init'''
+        raise NotImplementedError
+
+    def apply_rules(self, source_addr, rules):
+        '''Apply rules in given source address'''
+        raise NotImplementedError
+
+    def read_rules(self, target):
+        '''Read rules from QubesDB and return them as a list of dicts'''
+        entries = self.qdb.multiread('/qubes-firewall/{}/'.format(target))
+        assert isinstance(entries, dict)
+        # drop full path
+        entries = dict(((k.split('/')[3], v) for k, v in entries.items()))
+        if 'policy' not in entries:
+            raise RuleParseError('No \'policy\' defined')
+        policy = entries.pop('policy')
+        rules = []
+        for ruleno, rule in sorted(entries.items()):
+            if len(ruleno) != 4 or not ruleno.isdigit():
+                raise RuleParseError(
+                    'Unexpected non-rule found: {}={}'.format(ruleno, rule))
+            rule_dict = dict(elem.split('=') for elem in rule.split(' '))
+            if 'action' not in rule_dict:
+                raise RuleParseError('Rule \'{}\' lack action'.format(rule))
+            rules.append(rule_dict)
+        rules.append({'action': policy})
+        return rules
+
+    def list_targets(self):
+        return set(t.split('/')[2] for t in self.qdb.list('/qubes-firewall/'))
+
+    @staticmethod
+    def is_ip6(addr):
+        return addr.count(':') > 0
+
+    def log_error(self, msg):
+        self.log.error(msg)
+        subprocess.call(
+            ['notify-send', '-t', '3000', msg],
+            env=os.environ.copy().update({'DISPLAY': ':0'})
+        )
+
+    def handle_addr(self, addr):
+        try:
+            rules = self.read_rules(addr)
+            self.apply_rules(addr, rules)
+        except RuleParseError as e:
+            self.log_error(
+                'Failed to parse rules for {} ({}), blocking traffic'.format(
+                    addr, str(e)
+                ))
+            self.apply_rules(addr, [{'action': 'drop'}])
+        except RuleApplyError as e:
+            self.log_error(
+                'Failed to apply rules for {} ({}), blocking traffic'.format(
+                    addr, str(e))
+            )
+            # retry with fallback rules
+            try:
+                self.apply_rules(addr, [{'action': 'drop'}])
+            except RuleApplyError:
+                self.log_error(
+                    'Failed to block traffic for {}'.format(addr))
+
+    @staticmethod
+    def dns_addresses(family=None):
+        with open('/etc/resolv.conf') as resolv:
+            for line in resolv.readlines():
+                line = line.strip()
+                if line.startswith('nameserver'):
+                    if line.count('.') == 3 and (family or 4) == 4:
+                        yield line.split(' ')[1]
+                    elif line.count(':') and (family or 6) == 6:
+                        yield line.split(' ')[1]
+
+    def main(self):
+        self.terminate_requested = False
+        self.init()
+        # initial load
+        for source_addr in self.list_targets():
+            self.handle_addr(source_addr)
+        self.qdb.watch('/qubes-firewall/')
+        try:
+            for watch_path in iter(self.qdb.read_watch, None):
+                # ignore writing rules itself - wait for final write at
+                # source_addr level empty write (/qubes-firewall/SOURCE_ADDR)
+                if watch_path.count('/') > 2:
+                    continue
+                source_addr = watch_path.split('/')[2]
+                self.handle_addr(source_addr)
+        except OSError:  # EINTR
+            # signal received, don't continue the loop
+            pass
+
+        self.cleanup()
+
+    def terminate(self):
+        self.terminate_requested = True
+
+
+class IptablesWorker(FirewallWorker):
+    supported_rule_opts = ['action', 'proto', 'dst4', 'dst6', 'dsthost',
+        'dstports', 'specialtarget', 'icmptype']
+
+    def __init__(self):
+        super(IptablesWorker, self).__init__()
+        self.chains = {
+            4: set(),
+            6: set(),
+        }
+
+    @staticmethod
+    def chain_for_addr(addr):
+        '''Generate iptables chain name for given source address address'''
+        return 'qbs-' + addr.replace('.', '-').replace(':', '-')
+
+    def run_ipt(self, family, args, **kwargs):
+        # pylint: disable=no-self-use
+        if family == 6:
+            subprocess.check_call(['ip6tables'] + args, **kwargs)
+        else:
+            subprocess.check_call(['iptables'] + args, **kwargs)
+
+    def run_ipt_restore(self, family, args):
+        # pylint: disable=no-self-use
+        if family == 6:
+            return subprocess.Popen(['ip6tables-restore'] + args,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT)
+        else:
+            return subprocess.Popen(['iptables-restore'] + args,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT)
+
+    def create_chain(self, addr, chain, family):
+        '''
+        Create iptables chain and hook traffic coming from `addr` to it.
+
+        :param addr: source IP from which traffic should be handled by the
+        chain
+        :param chain: name of the chain to create
+        :param family: address family (4 or 6)
+        :return: None
+        '''
+
+        self.run_ipt(family, ['-N', chain])
+        self.run_ipt(family,
+            ['-A', 'QBS-FORWARD', '-s', addr, '-j', chain])
+        self.chains[family].add(chain)
+
+    def prepare_rules(self, chain, rules, family):
+        '''
+        Helper function to translate rules list into input for iptables-restore
+
+        :param chain: name of the chain to put rules into
+        :param rules: list of rules
+        :param family: address family (4 or 6)
+        :return: input for iptables-restore
+        :rtype: str
+        '''
+
+        iptables = "*filter\n"
+
+        fullmask = '/128' if family == 6 else '/32'
+
+        dns = list(addr + fullmask for addr in self.dns_addresses(family))
+
+        for rule in rules:
+            unsupported_opts = set(rule.keys()).difference(
+                set(self.supported_rule_opts))
+            if unsupported_opts:
+                raise RuleParseError(
+                    'Unsupported rule option(s): {!s}'.format(unsupported_opts))
+            if 'dst4' in rule and family == 6:
+                raise RuleParseError('IPv4 rule found for IPv6 address')
+            if 'dst6' in rule and family == 4:
+                raise RuleParseError('dst6 rule found for IPv4 address')
+
+            if 'proto' in rule:
+                protos = [rule['proto']]
+            else:
+                protos = None
+
+            if 'dst4' in rule:
+                dsthosts = [rule['dst4']]
+            elif 'dst6' in rule:
+                dsthosts = [rule['dst6']]
+            elif 'dsthost' in rule:
+                addrinfo = socket.getaddrinfo(rule['dsthost'], None,
+                    (socket.AF_INET6 if family == 6 else socket.AF_INET))
+                dsthosts = set(item[4][0] + fullmask for item in addrinfo)
+            else:
+                dsthosts = None
+
+            if 'dstports' in rule:
+                dstports = rule['dstports'].replace('-', ':')
+            else:
+                dstports = None
+
+            if rule.get('specialtarget', None) == 'dns':
+                if dstports not in ('53:53', None):
+                    continue
+                else:
+                    dstports = '53:53'
+                if not dns:
+                    continue
+                if protos is not None:
+                    protos = {'tcp', 'udp'}.intersection(protos)
+                else:
+                    protos = {'tcp', 'udp'}
+
+                if dsthosts is not None:
+                    dsthosts = set(dns).intersection(dsthosts)
+                else:
+                    dsthosts = dns
+
+            if 'icmptype' in rule:
+                icmptype = rule['icmptype']
+            else:
+                icmptype = None
+
+            # make them iterable
+            if protos is None:
+                protos = [None]
+            if dsthosts is None:
+                dsthosts = [None]
+
+            # sorting here is only to ease writing tests
+            for proto in sorted(protos):
+                for dsthost in sorted(dsthosts):
+                    ipt_rule = '-A {}'.format(chain)
+                    if dsthost is not None:
+                        ipt_rule += ' -d {}'.format(dsthost)
+                    if proto is not None:
+                        ipt_rule += ' -p {}'.format(proto)
+                    if dstports is not None:
+                        ipt_rule += ' --dport {}'.format(dstports)
+                    if icmptype is not None:
+                        ipt_rule += ' --icmp-type {}'.format(icmptype)
+                    ipt_rule += ' -j {}\n'.format(
+                        str(rule['action']).upper())
+                    iptables += ipt_rule
+
+        iptables += 'COMMIT\n'
+        return iptables
+
+    def apply_rules_family(self, source, rules, family):
+        '''
+        Apply rules for given source address.
+        Handle only rules for given address family (IPv4 or IPv6).
+
+        :param source: source address
+        :param rules: rules list
+        :param family: address family, either 4 or 6
+        :return: None
+        '''
+
+        chain = self.chain_for_addr(source)
+        if chain not in self.chains[family]:
+            self.create_chain(source, chain, family)
+
+        iptables = self.prepare_rules(chain, rules, family)
+        try:
+            self.run_ipt(family, ['-F', chain])
+            p = self.run_ipt_restore(family, ['-n'])
+            (output, _) = p.communicate(iptables)
+            if p.returncode != 0:
+                raise RuleApplyError(
+                    'iptables-restore failed: {}'.format(output))
+        except subprocess.CalledProcessError as e:
+            raise RuleApplyError('\'iptables -F {}\' failed: {}'.format(
+                chain, e.output))
+
+    def apply_rules(self, source, rules):
+        if self.is_ip6(source):
+            self.apply_rules_family(source, rules, 6)
+        else:
+            self.apply_rules_family(source, rules, 4)
+
+    def init(self):
+        # make sure 'QBS_FORWARD' chain exists - should be created before
+        # starting qubes-firewall
+        try:
+            self.run_ipt(4, ['-nL', 'QBS-FORWARD'])
+            self.run_ipt(6, ['-nL', 'QBS-FORWARD'])
+        except subprocess.CalledProcessError:
+            self.log_error('\'QBS-FORWARD\' chain not found, create it first')
+            sys.exit(1)
+
+    def cleanup(self):
+        for family in (4, 6):
+            self.run_ipt(family, ['-F', 'QBS-FORWARD'])
+            for chain in self.chains[family]:
+                self.run_ipt(family, ['-F', chain])
+                self.run_ipt(family, ['-X', chain])
+
+
+class NftablesWorker(FirewallWorker):
+    supported_rule_opts = ['action', 'proto', 'dst4', 'dst6', 'dsthost',
+        'dstports', 'specialtarget', 'icmptype']
+
+    def __init__(self):
+        super(NftablesWorker, self).__init__()
+        self.chains = {
+            4: set(),
+            6: set(),
+        }
+
+    @staticmethod
+    def chain_for_addr(addr):
+        '''Generate iptables chain name for given source address address'''
+        return 'qbs-' + addr.replace('.', '-').replace(':', '-')
+
+    def run_nft(self, nft_input):
+        # pylint: disable=no-self-use
+        p = subprocess.Popen(['nft', '-f', '/dev/stdin'],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT)
+        stdout, _ = p.communicate(nft_input)
+        if p.returncode != 0:
+            raise RuleApplyError('nft failed: {}'.format(stdout))
+
+    def create_chain(self, addr, chain, family):
+        '''
+        Create iptables chain and hook traffic coming from `addr` to it.
+
+        :param addr: source IP from which traffic should be handled by the
+        chain
+        :param chain: name of the chain to create
+        :param family: address family (4 or 6)
+        :return: None
+        '''
+        nft_input = (
+            'table {family} {table} {{\n'
+            '  chain {chain} {{\n'
+            '  }}\n'
+            '  chain forward {{\n'
+            '    {family} saddr {ip} jump {chain}\n'
+            '  }}\n'
+            '}}\n'.format(
+                family=("ip6" if family == 6 else "ip"),
+                table='qubes-firewall',
+                chain=chain,
+                ip=addr,
+            )
+        )
+        self.run_nft(nft_input)
+        self.chains[family].add(chain)
+
+    def prepare_rules(self, chain, rules, family):
+        '''
+        Helper function to translate rules list into input for iptables-restore
+
+        :param chain: name of the chain to put rules into
+        :param rules: list of rules
+        :param family: address family (4 or 6)
+        :return: input for iptables-restore
+        :rtype: str
+        '''
+
+        assert family in (4, 6)
+        nft_rules = []
+        ip_match = 'ip6' if family == 6 else 'ip'
+
+        fullmask = '/128' if family == 6 else '/32'
+
+        dns = list(addr + fullmask for addr in self.dns_addresses(family))
+
+        for rule in rules:
+            unsupported_opts = set(rule.keys()).difference(
+                set(self.supported_rule_opts))
+            if unsupported_opts:
+                raise RuleParseError(
+                    'Unsupported rule option(s): {!s}'.format(unsupported_opts))
+            if 'dst4' in rule and family == 6:
+                raise RuleParseError('IPv4 rule found for IPv6 address')
+            if 'dst6' in rule and family == 4:
+                raise RuleParseError('dst6 rule found for IPv4 address')
+
+            nft_rule = ""
+
+            if 'proto' in rule:
+                if family == 4:
+                    nft_rule += ' ip protocol {}'.format(rule['proto'])
+                elif family == 6:
+                    proto = 'icmpv6' if rule['proto'] == 'icmp' \
+                        else rule['proto']
+                    nft_rule += ' ip6 nexthdr {}'.format(proto)
+
+
+            if 'dst4' in rule:
+                nft_rule += ' ip daddr {}'.format(rule['dst4'])
+            elif 'dst6' in rule:
+                nft_rule += ' ip6 daddr {}'.format(rule['dst6'])
+            elif 'dsthost' in rule:
+                addrinfo = socket.getaddrinfo(rule['dsthost'], None,
+                    (socket.AF_INET6 if family == 6 else socket.AF_INET))
+                nft_rule += ' {} daddr {{ {} }}'.format(ip_match,
+                    ', '.join(set(item[4][0] + fullmask for item in addrinfo)))
+
+            if 'dstports' in rule:
+                dstports = rule['dstports']
+                if len(set(dstports.split('-'))) == 1:
+                    dstports = dstports.split('-')[0]
+            else:
+                dstports = None
+
+            if rule.get('specialtarget', None) == 'dns':
+                if dstports not in ('53', None):
+                    continue
+                else:
+                    dstports = '53'
+                if not dns:
+                    continue
+                nft_rule += ' {} daddr {{ {} }}'.format(ip_match, ', '.join(
+                    dns))
+
+            if 'icmptype' in rule:
+                if family == 4:
+                    nft_rule += ' icmp type {}'.format(rule['icmptype'])
+                elif family == 6:
+                    nft_rule += ' icmpv6 type {}'.format(rule['icmptype'])
+
+            # now duplicate rules for tcp/udp if needed
+            # it isn't possible to specify "tcp dport xx || udp dport xx" in
+            # one rule
+            if dstports is not None:
+                if 'proto' not in rule:
+                    nft_rules.append(
+                        nft_rule + ' tcp dport {} {}'.format(
+                            dstports, rule['action']))
+                    nft_rules.append(
+                        nft_rule + ' udp dport {} {}'.format(
+                            dstports, rule['action']))
+                else:
+                    nft_rules.append(
+                        nft_rule + ' {} dport {} {}'.format(
+                            rule['proto'], dstports, rule['action']))
+            else:
+                nft_rules.append(nft_rule + ' ' + rule['action'])
+
+        return (
+            'flush chain {family} {table} {chain}\n'
+            'table {family} {table} {{\n'
+            '  chain {chain} {{\n'
+            '   {rules}\n'
+            '  }}\n'
+            '}}\n'.format(
+                family=('ip6' if family == 6 else 'ip'),
+                table='qubes-firewall',
+                chain=chain,
+                rules='\n   '.join(nft_rules)
+            ))
+
+    def apply_rules_family(self, source, rules, family):
+        '''
+        Apply rules for given source address.
+        Handle only rules for given address family (IPv4 or IPv6).
+
+        :param source: source address
+        :param rules: rules list
+        :param family: address family, either 4 or 6
+        :return: None
+        '''
+
+        chain = self.chain_for_addr(source)
+        if chain not in self.chains[family]:
+            self.create_chain(source, chain, family)
+
+        self.run_nft(self.prepare_rules(chain, rules, family))
+
+    def apply_rules(self, source, rules):
+        if self.is_ip6(source):
+            self.apply_rules_family(source, rules, 6)
+        else:
+            self.apply_rules_family(source, rules, 4)
+
+    def init(self):
+        # make sure 'QBS_FORWARD' chain exists - should be created before
+        # starting qubes-firewall
+        nft_init = (
+            'table {family} qubes-firewall {{\n'
+            '  chain forward {{\n'
+            '    type filter hook forward priority 0;\n'
+            '  }}\n'
+            '}}\n'
+        )
+        nft_init = ''.join(
+            nft_init.format(family=family) for family in ('ip', 'ip6'))
+        self.run_nft(nft_init)
+
+    def cleanup(self):
+        nft_cleanup = (
+            'delete table ip qubes-firewall\n'
+            'delete table ip6 qubes-firewall\n'
+        )
+        self.run_nft(nft_cleanup)
+
+
+def main():
+    if spawn.find_executable('nft'):
+        worker = NftablesWorker()
+    else:
+        worker = IptablesWorker()
+    context = daemon.DaemonContext()
+    context.stderr = sys.stderr
+    context.detach_process = False
+    context.files_preserve = [worker.qdb.watch_fd()]
+    context.signal_map = {
+        signal.SIGTERM: lambda _signal, _stack: worker.terminate(),
+    }
+    with context:
+        worker.main()
+
+if __name__ == '__main__':
+    main()

qubesagent/test_firewall.py

@@ -0,0 +1,527 @@
+import logging
+import operator
+from unittest import TestCase
+from unittest.mock import patch
+
+import qubesagent.firewall
+
+
+class DummyIptablesRestore(object):
+    # pylint: disable=too-few-public-methods
+    def __init__(self, worker_mock, family):
+        self._worker_mock = worker_mock
+        self._family = family
+        self.returncode = 0
+
+    def communicate(self, stdin=None):
+        self._worker_mock.loaded_iptables[self._family] = stdin
+        return ("", None)
+
+class DummyQubesDB(object):
+    def __init__(self, worker_mock):
+        self._worker_mock = worker_mock
+        self.entries = {}
+        self.pending_watches = []
+
+    def read(self, key):
+        try:
+            return self.entries[key]
+        except KeyError:
+            return None
+
+    def multiread(self, prefix):
+        result = {}
+        for key, value in self.entries.items():
+            if key.startswith(prefix):
+                result[key] = value
+        return result
+
+    def list(self, prefix):
+        result = []
+        for key in self.entries.keys():
+            if key.startswith(prefix):
+                result.append(key)
+        return result
+
+    def watch(self, path):
+        pass
+
+    def read_watch(self):
+        try:
+            return self.pending_watches.pop(0)
+        except IndexError:
+            return None
+
+
+class FirewallWorker(qubesagent.firewall.FirewallWorker):
+    def __init__(self):
+        # pylint: disable=super-init-not-called
+        # don't call super on purpose - avoid connecting to QubesDB
+        # super(FirewallWorker, self).__init__()
+        self.qdb = DummyQubesDB(self)
+        self.log = logging.getLogger('qubes.tests')
+
+        self.init_called = False
+        self.cleanup_called = False
+        self.rules = {}
+
+    def apply_rules(self, source_addr, rules):
+        self.rules[source_addr] = rules
+
+    def cleanup(self):
+        self.init_called = True
+
+    def init(self):
+        self.cleanup_called = True
+
+
+class IptablesWorker(qubesagent.firewall.IptablesWorker):
+    '''Override methods actually modifying system state to only log what
+    would be done'''
+
+    def __init__(self):
+        # pylint: disable=super-init-not-called
+        # don't call super on purpose - avoid connecting to QubesDB
+        # super(IptablesWorker, self).__init__()
+        # copied __init__:
+        self.qdb = DummyQubesDB(self)
+        self.log = logging.getLogger('qubes.tests')
+        self.chains = {
+            4: set(),
+            6: set(),
+        }
+
+        #: instead of really running `iptables`, log what would be called
+        self.called_commands = {
+            4: [],
+            6: [],
+        }
+        #: rules that would be loaded with `iptables-restore`
+        self.loaded_iptables = {
+            4: None,
+            6: None,
+        }
+
+    def run_ipt(self, family, args, **kwargs):
+        self.called_commands[family].append(args)
+
+    def run_ipt_restore(self, family, args):
+        return DummyIptablesRestore(self, family)
+
+    @staticmethod
+    def dns_addresses(family=None):
+        if family == 4:
+            return ['1.1.1.1', '2.2.2.2']
+        else:
+            return ['2001::1', '2001::2']
+
+
+class NftablesWorker(qubesagent.firewall.NftablesWorker):
+    '''Override methods actually modifying system state to only log what
+    would be done'''
+
+    def __init__(self):
+        # pylint: disable=super-init-not-called
+        # don't call super on purpose - avoid connecting to QubesDB
+        # super(IptablesWorker, self).__init__()
+        # copied __init__:
+        self.qdb = DummyQubesDB(self)
+        self.log = logging.getLogger('qubes.tests')
+        self.chains = {
+            4: set(),
+            6: set(),
+        }
+
+        #: instead of really running `nft`, log what would be loaded
+        #: rules that would be loaded with `nft`
+        self.loaded_rules = []
+
+    def run_nft(self, nft_input):
+        self.loaded_rules.append(nft_input)
+
+    @staticmethod
+    def dns_addresses(family=None):
+        if family == 4:
+            return ['1.1.1.1', '2.2.2.2']
+        else:
+            return ['2001::1', '2001::2']
+
+
+class TestIptablesWorker(TestCase):
+    def setUp(self):
+        super(TestIptablesWorker, self).setUp()
+        self.obj = IptablesWorker()
+        self.subprocess_patch = patch('subprocess.call')
+        self.subprocess_mock = self.subprocess_patch.start()
+
+    def tearDown(self):
+        self.subprocess_patch.stop()
+
+    def test_000_chain_for_addr(self):
+        self.assertEqual(
+            self.obj.chain_for_addr('10.137.0.1'), 'qbs-10-137-0-1')
+        self.assertEqual(
+            self.obj.chain_for_addr('fd09:24ef:4179:0000::3'),
+            'qbs-fd09-24ef-4179-0000--3')
+
+    def test_001_create_chain(self):
+        testdata = [
+            (4, '10.137.0.1', 'qbs-10-137-0-1'),
+            (6, 'fd09:24ef:4179:0000::3', 'qbs-fd09-24ef-4179-0000--3')
+        ]
+        for family, addr, chain in testdata:
+            self.obj.create_chain(addr, chain, family)
+            self.assertEqual(self.obj.called_commands[family],
+                [['-N', chain],
+                    ['-A', 'QBS-FORWARD', '-s', addr, '-j', chain]])
+
+    def test_002_prepare_rules4(self):
+        rules = [
+            {'action': 'accept', 'proto': 'tcp',
+                'dstports': '80-80', 'dst4': '1.2.3.0/24'},
+            {'action': 'accept', 'proto': 'udp',
+                'dstports': '443-1024', 'dsthost': 'yum.qubes-os.org'},
+            {'action': 'accept', 'specialtarget': 'dns'},
+            {'action': 'drop', 'proto': 'udp', 'specialtarget': 'dns'},
+            {'action': 'drop', 'proto': 'icmp'},
+            {'action': 'drop'},
+        ]
+        expected_iptables = (
+            "*filter\n"
+            "-A chain -d 1.2.3.0/24 -p tcp --dport 80:80 -j ACCEPT\n"
+            "-A chain -d 82.94.215.165/32 -p udp --dport 443:1024 -j ACCEPT\n"
+            "-A chain -d 1.1.1.1/32 -p tcp --dport 53:53 -j ACCEPT\n"
+            "-A chain -d 2.2.2.2/32 -p tcp --dport 53:53 -j ACCEPT\n"
+            "-A chain -d 1.1.1.1/32 -p udp --dport 53:53 -j ACCEPT\n"
+            "-A chain -d 2.2.2.2/32 -p udp --dport 53:53 -j ACCEPT\n"
+            "-A chain -d 1.1.1.1/32 -p udp --dport 53:53 -j DROP\n"
+            "-A chain -d 2.2.2.2/32 -p udp --dport 53:53 -j DROP\n"
+            "-A chain -p icmp -j DROP\n"
+            "-A chain -j DROP\n"
+            "COMMIT\n"
+        )
+        self.assertEqual(self.obj.prepare_rules('chain', rules, 4),
+            expected_iptables)
+        with self.assertRaises(qubesagent.firewall.RuleParseError):
+            self.obj.prepare_rules('chain', [{'unknown': 'xxx'}], 4)
+        with self.assertRaises(qubesagent.firewall.RuleParseError):
+            self.obj.prepare_rules('chain', [{'dst6': 'a::b'}], 4)
+        with self.assertRaises(qubesagent.firewall.RuleParseError):
+            self.obj.prepare_rules('chain', [{'dst4': '3.3.3.3'}], 6)
+
+    def test_003_prepare_rules6(self):
+        rules = [
+            {'action': 'accept', 'proto': 'tcp',
+                'dstports': '80-80', 'dst6': 'a::b/128'},
+            {'action': 'accept', 'proto': 'tcp',
+                'dsthost': 'ripe.net'},
+            {'action': 'accept', 'specialtarget': 'dns'},
+            {'action': 'drop', 'proto': 'udp', 'specialtarget': 'dns'},
+            {'action': 'drop', 'proto': 'icmp'},
+            {'action': 'drop'},
+        ]
+        expected_iptables = (
+            "*filter\n"
+            "-A chain -d a::b/128 -p tcp --dport 80:80 -j ACCEPT\n"
+            "-A chain -d 2001:67c:2e8:22::c100:68b/128 -p tcp -j ACCEPT\n"
+            "-A chain -d 2001::1/128 -p tcp --dport 53:53 -j ACCEPT\n"
+            "-A chain -d 2001::2/128 -p tcp --dport 53:53 -j ACCEPT\n"
+            "-A chain -d 2001::1/128 -p udp --dport 53:53 -j ACCEPT\n"
+            "-A chain -d 2001::2/128 -p udp --dport 53:53 -j ACCEPT\n"
+            "-A chain -d 2001::1/128 -p udp --dport 53:53 -j DROP\n"
+            "-A chain -d 2001::2/128 -p udp --dport 53:53 -j DROP\n"
+            "-A chain -p icmp -j DROP\n"
+            "-A chain -j DROP\n"
+            "COMMIT\n"
+        )
+        self.assertEqual(self.obj.prepare_rules('chain', rules, 6),
+            expected_iptables)
+
+    def test_004_apply_rules4(self):
+        rules = [{'action': 'accept'}]
+        chain = 'qbs-10-137-0-1'
+        self.obj.apply_rules('10.137.0.1', rules)
+        self.assertEqual(self.obj.called_commands[4],
+            [
+                ['-N', chain],
+                ['-A', 'QBS-FORWARD', '-s', '10.137.0.1', '-j', chain],
+                ['-F', chain]])
+        self.assertEqual(self.obj.loaded_iptables[4],
+            self.obj.prepare_rules(chain, rules, 4))
+        self.assertEqual(self.obj.called_commands[6], [])
+        self.assertIsNone(self.obj.loaded_iptables[6])
+
+    def test_005_apply_rules6(self):
+        rules = [{'action': 'accept'}]
+        chain = 'qbs-2000--a'
+        self.obj.apply_rules('2000::a', rules)
+        self.assertEqual(self.obj.called_commands[6],
+            [
+                ['-N', chain],
+                ['-A', 'QBS-FORWARD', '-s', '2000::a', '-j', chain],
+                ['-F', chain]])
+        self.assertEqual(self.obj.loaded_iptables[6],
+            self.obj.prepare_rules(chain, rules, 6))
+        self.assertEqual(self.obj.called_commands[4], [])
+        self.assertIsNone(self.obj.loaded_iptables[4])
+
+    def test_006_init(self):
+        self.obj.init()
+        self.assertEqual(self.obj.called_commands[4],
+            [['-nL', 'QBS-FORWARD']])
+        self.assertEqual(self.obj.called_commands[6],
+            [['-nL', 'QBS-FORWARD']])
+
+    def test_007_cleanup(self):
+        self.obj.init()
+        self.obj.create_chain('1.2.3.4', 'chain-ip4-1', 4)
+        self.obj.create_chain('1.2.3.6', 'chain-ip4-2', 4)
+        self.obj.create_chain('2000::1', 'chain-ip6-1', 6)
+        self.obj.create_chain('2000::2', 'chain-ip6-2', 6)
+        # forget about commands called earlier
+        self.obj.called_commands[4] = []
+        self.obj.called_commands[6] = []
+        self.obj.cleanup()
+        self.assertEqual([self.obj.called_commands[4][0]] +
+                sorted(self.obj.called_commands[4][1:], key=operator.itemgetter(1)),
+            [['-F', 'QBS-FORWARD'],
+                ['-F', 'chain-ip4-1'],
+                ['-X', 'chain-ip4-1'],
+                ['-F', 'chain-ip4-2'],
+                ['-X', 'chain-ip4-2']])
+        self.assertEqual([self.obj.called_commands[6][0]] +
+                sorted(self.obj.called_commands[6][1:], key=operator.itemgetter(1)),
+            [['-F', 'QBS-FORWARD'],
+                ['-F', 'chain-ip6-1'],
+                ['-X', 'chain-ip6-1'],
+                ['-F', 'chain-ip6-2'],
+                ['-X', 'chain-ip6-2']])
+
+
+class TestNftablesWorker(TestCase):
+    def setUp(self):
+        super(TestNftablesWorker, self).setUp()
+        self.obj = NftablesWorker()
+        self.subprocess_patch = patch('subprocess.call')
+        self.subprocess_mock = self.subprocess_patch.start()
+
+    def tearDown(self):
+        self.subprocess_patch.stop()
+
+    def test_000_chain_for_addr(self):
+        self.assertEqual(
+            self.obj.chain_for_addr('10.137.0.1'), 'qbs-10-137-0-1')
+        self.assertEqual(
+            self.obj.chain_for_addr('fd09:24ef:4179:0000::3'),
+            'qbs-fd09-24ef-4179-0000--3')
+
+    def expected_create_chain(self, family, addr, chain):
+        return (
+            'table {family} qubes-firewall {{\n'
+            '  chain {chain} {{\n'
+            '  }}\n'
+            '  chain forward {{\n'
+            '    {family} saddr {addr} jump {chain}\n'
+            '  }}\n'
+            '}}\n'.format(family=family, addr=addr, chain=chain))
+
+    def test_001_create_chain(self):
+        testdata = [
+            (4, '10.137.0.1', 'qbs-10-137-0-1'),
+            (6, 'fd09:24ef:4179:0000::3', 'qbs-fd09-24ef-4179-0000--3')
+        ]
+        for family, addr, chain in testdata:
+            self.obj.create_chain(addr, chain, family)
+        self.assertEqual(self.obj.loaded_rules,
+            [self.expected_create_chain('ip', '10.137.0.1', 'qbs-10-137-0-1'),
+             self.expected_create_chain(
+                 'ip6', 'fd09:24ef:4179:0000::3', 'qbs-fd09-24ef-4179-0000--3'),
+             ])
+
+    def test_002_prepare_rules4(self):
+        rules = [
+            {'action': 'accept', 'proto': 'tcp',
+                'dstports': '80-80', 'dst4': '1.2.3.0/24'},
+            {'action': 'accept', 'proto': 'udp',
+                'dstports': '443-1024', 'dsthost': 'yum.qubes-os.org'},
+            {'action': 'accept', 'specialtarget': 'dns'},
+            {'action': 'drop', 'proto': 'udp', 'specialtarget': 'dns'},
+            {'action': 'drop', 'proto': 'icmp'},
+            {'action': 'drop'},
+        ]
+        expected_nft = (
+            'flush chain ip qubes-firewall chain\n'
+            'table ip qubes-firewall {\n'
+            '  chain chain {\n'
+            '    ip protocol tcp ip daddr 1.2.3.0/24 tcp dport 80 accept\n'
+            '    ip protocol udp ip daddr { 82.94.215.165/32 } '
+            'udp dport 443-1024 accept\n'
+            '    ip daddr { 1.1.1.1/32, 2.2.2.2/32 } tcp dport 53 accept\n'
+            '    ip daddr { 1.1.1.1/32, 2.2.2.2/32 } udp dport 53 accept\n'
+            '    ip protocol udp ip daddr { 1.1.1.1/32, 2.2.2.2/32 } udp dport '
+            '53 drop\n'
+            '    ip protocol icmp drop\n'
+            '    drop\n'
+            '  }\n'
+            '}\n'
+        )
+        self.assertEqual(self.obj.prepare_rules('chain', rules, 4),
+            expected_nft)
+        with self.assertRaises(qubesagent.firewall.RuleParseError):
+            self.obj.prepare_rules('chain', [{'unknown': 'xxx'}], 4)
+        with self.assertRaises(qubesagent.firewall.RuleParseError):
+            self.obj.prepare_rules('chain', [{'dst6': 'a::b'}], 4)
+        with self.assertRaises(qubesagent.firewall.RuleParseError):
+            self.obj.prepare_rules('chain', [{'dst4': '3.3.3.3'}], 6)
+
+    def test_003_prepare_rules6(self):
+        rules = [
+            {'action': 'accept', 'proto': 'tcp',
+                'dstports': '80-80', 'dst6': 'a::b/128'},
+            {'action': 'accept', 'proto': 'tcp',
+                'dsthost': 'ripe.net'},
+            {'action': 'accept', 'specialtarget': 'dns'},
+            {'action': 'drop', 'proto': 'udp', 'specialtarget': 'dns'},
+            {'action': 'drop', 'proto': 'icmp', 'icmptype': '128'},
+            {'action': 'drop'},
+        ]
+        expected_nft = (
+            'flush chain ip6 qubes-firewall chain\n'
+            'table ip6 qubes-firewall {\n'
+            '  chain chain {\n'
+            '    ip6 nexthdr tcp ip6 daddr a::b/128 tcp dport 80 accept\n'
+            '    ip6 nexthdr tcp ip6 daddr { 2001:67c:2e8:22::c100:68b/128 } '
+            'accept\n'
+            '    ip6 daddr { 2001::1/128, 2001::2/128 } tcp dport 53 accept\n'
+            '    ip6 daddr { 2001::1/128, 2001::2/128 } udp dport 53 accept\n'
+            '    ip6 nexthdr udp ip6 daddr { 2001::1/128, 2001::2/128 } '
+            'udp dport 53 drop\n'
+            '    ip6 nexthdr icmpv6 icmpv6 type 128 drop\n'
+            '    drop\n'
+            '  }\n'
+            '}\n'
+        )
+        self.assertEqual(self.obj.prepare_rules('chain', rules, 6),
+            expected_nft)
+
+    def test_004_apply_rules4(self):
+        rules = [{'action': 'accept'}]
+        chain = 'qbs-10-137-0-1'
+        self.obj.apply_rules('10.137.0.1', rules)
+        self.assertEqual(self.obj.loaded_rules,
+            [self.expected_create_chain('ip', '10.137.0.1', chain),
+             self.obj.prepare_rules(chain, rules, 4),
+             ])
+
+    def test_005_apply_rules6(self):
+        rules = [{'action': 'accept'}]
+        chain = 'qbs-2000--a'
+        self.obj.apply_rules('2000::a', rules)
+        self.assertEqual(self.obj.loaded_rules,
+            [self.expected_create_chain('ip6', '2000::a', chain),
+             self.obj.prepare_rules(chain, rules, 6),
+             ])
+
+    def test_006_init(self):
+        self.obj.init()
+        self.assertEqual(self.obj.loaded_rules,
+        [
+            'table ip qubes-firewall {\n'
+            '  chain forward {\n'
+            '    type filter hook forward priority 0;\n'
+            '  }\n'
+            '}\n'
+            'table ip6 qubes-firewall {\n'
+            '  chain forward {\n'
+            '    type filter hook forward priority 0;\n'
+            '  }\n'
+            '}\n'
+        ])
+
+    def test_007_cleanup(self):
+        self.obj.init()
+        self.obj.create_chain('1.2.3.4', 'chain-ip4-1', 4)
+        self.obj.create_chain('1.2.3.6', 'chain-ip4-2', 4)
+        self.obj.create_chain('2000::1', 'chain-ip6-1', 6)
+        self.obj.create_chain('2000::2', 'chain-ip6-2', 6)
+        # forget about commands called earlier
+        self.obj.loaded_rules = []
+        self.obj.cleanup()
+        self.assertEqual(self.obj.loaded_rules,
+            ['delete table ip qubes-firewall\n'
+             'delete table ip6 qubes-firewall\n',
+             ])
+
+class TestFirewallWorker(TestCase):
+    def setUp(self):
+        self.obj = FirewallWorker()
+        rules = {
+            '10.137.0.1': {
+                'policy': 'accept',
+                '0000': 'proto=tcp dstports=80-80 action=drop',
+                '0001': 'proto=udp specialtarget=dns action=accept',
+                '0002': 'proto=udp action=drop',
+            },
+            '10.137.0.2': {'policy': 'accept'},
+            # no policy
+            '10.137.0.3': {'0000': 'proto=tcp action=accept'},
+            # no action
+            '10.137.0.4': {
+                'policy': 'drop',
+                '0000': 'proto=tcp'
+            },
+        }
+        for addr, entries in rules.items():
+            for key, value in entries.items():
+                self.obj.qdb.entries[
+                    '/qubes-firewall/{}/{}'.format(addr, key)] = value
+
+        self.subprocess_patch = patch('subprocess.call')
+        self.subprocess_mock = self.subprocess_patch.start()
+
+    def tearDown(self):
+        self.subprocess_patch.stop()
+
+    def test_read_rules(self):
+        expected_rules1 = [
+            {'proto': 'tcp', 'dstports': '80-80', 'action': 'drop'},
+            {'proto': 'udp', 'specialtarget': 'dns', 'action': 'accept'},
+            {'proto': 'udp', 'action': 'drop'},
+            {'action': 'accept'},
+        ]
+        expected_rules2 = [
+            {'action': 'accept'},
+        ]
+        self.assertEqual(self.obj.read_rules('10.137.0.1'), expected_rules1)
+        self.assertEqual(self.obj.read_rules('10.137.0.2'), expected_rules2)
+        with self.assertRaises(qubesagent.firewall.RuleParseError):
+            self.obj.read_rules('10.137.0.3')
+        with self.assertRaises(qubesagent.firewall.RuleParseError):
+            self.obj.read_rules('10.137.0.4')
+
+
+    def test_list_targets(self):
+        self.assertEqual(self.obj.list_targets(), set(['10.137.0.{}'.format(x)
+            for x in range(1, 5)]))
+
+    def test_is_ip6(self):
+        self.assertTrue(self.obj.is_ip6('2000::abcd'))
+        self.assertTrue(self.obj.is_ip6('2000:1:2:3:4:5:6:abcd'))
+        self.assertFalse(self.obj.is_ip6('10.137.0.1'))
+
+    def test_handle_addr(self):
+        self.obj.handle_addr('10.137.0.2')
+        self.assertEqual(self.obj.rules['10.137.0.2'], [{'action': 'accept'}])
+        # fallback to block all
+        self.obj.handle_addr('10.137.0.3')
+        self.assertEqual(self.obj.rules['10.137.0.3'], [{'action': 'drop'}])
+        self.obj.handle_addr('10.137.0.4')
+        self.assertEqual(self.obj.rules['10.137.0.4'], [{'action': 'drop'}])
+
+
+    def test_main(self):
+        self.obj.main()
+        self.assertTrue(self.obj.init_called)
+        self.assertTrue(self.obj.cleanup_called)
+        self.assertEqual(set(self.obj.rules.keys()), self.obj.list_targets())
+        # rules content were already tested

rpm_spec/core-vm.spec

@@ -20,7 +20,7 @@
 #
 #
 
-%define qubes_services qubes-core qubes-core-netvm qubes-core-early qubes-firewall qubes-netwatcher qubes-iptables qubes-updates-proxy qubes-qrexec-agent qubes-dvm
+%define qubes_services qubes-core qubes-core-netvm qubes-core-early qubes-firewall qubes-iptables qubes-updates-proxy qubes-qrexec-agent qubes-dvm
 %define qubes_preset_file 75-qubes-vm.preset
 
 %{!?version: %define version %(cat version)}
@@ -139,6 +139,8 @@ Requires:   pygobject3-base
 Requires:   dbus-python
 # for qubes-session-autostart, xdg-icon
 Requires:   pyxdg
+Requires:   python-daemon
+Requires:   nftables
 Requires:   ImageMagick
 Requires:   librsvg2-tools
 Requires:   fakeroot
@@ -157,6 +159,7 @@ Requires:   python2-dnf-plugins-qubes-hooks
 Obsoletes:  qubes-core-vm-kernel-placeholder <= 1.0
 Obsoletes:  qubes-upgrade-vm < 3.2
 BuildRequires: xen-devel
+BuildRequires: python3-devel
 BuildRequires: libX11-devel
 BuildRequires: qubes-utils-devel >= 3.1.3
 BuildRequires: qubes-libvchan-%{backend_vmm}-devel
@@ -430,6 +433,8 @@ rm -f %{name}-%{version}
 %config(noreplace) /etc/qubes-rpc/qubes.GetImageRGBA
 %config(noreplace) /etc/qubes-rpc/qubes.SetDateTime
 %config(noreplace) /etc/qubes-rpc/qubes.InstallUpdatesGUI
+%config(noreplace) /etc/qubes-rpc/qubes.ResizeDisk
+%config(noreplace) /etc/qubes-rpc/qubes.StartApp
 %dir /etc/qubes/autostart
 /etc/qubes/autostart/README.txt
 %config /etc/qubes/autostart/*.desktop.d/30_qubes.conf
@@ -449,6 +454,7 @@ rm -f %{name}-%{version}
 %config(noreplace) /etc/qubes-suspend-module-blacklist
 /etc/xdg/autostart/00-qubes-show-hide-nm-applet.desktop
 /etc/xen/scripts/vif-route-qubes
+/etc/xen/scripts/vif-qubes-nat.sh
 %config(noreplace) /etc/yum.conf.d/qubes-proxy.conf
 %config(noreplace) /etc/yum.repos.d/qubes-r3.repo
 /etc/yum/pluginconf.d/yum-qubes-hooks.conf
@@ -464,6 +470,7 @@ rm -f %{name}-%{version}
 /usr/bin/qvm-open-in-vm
 /usr/bin/qvm-run
 /usr/bin/qvm-mru-entry
+/usr/bin/qvm-features-request
 /usr/bin/xenstore-watch-qubes
 /usr/bin/qubes-desktop-run
 /usr/bin/qubes-open
@@ -510,17 +517,25 @@ rm -f %{name}-%{version}
 /usr/lib/qubes/init/functions
 %dir /usr/lib/qubes-bind-dirs.d
 /usr/lib/qubes-bind-dirs.d/30_cron.conf
-/usr/lib64/python2.7/site-packages/qubes/xdg.py*
+/usr/lib/python2.7/site-packages/qubesxdg.py*
 /usr/sbin/qubes-firewall
-/usr/sbin/qubes-netwatcher
 /usr/share/qubes/serial.conf
-/usr/share/glib-2.0/schemas/org.gnome.settings-daemon.plugins.updates.gschema.override
-/usr/share/glib-2.0/schemas/org.gnome.nautilus.gschema.override
-/usr/share/glib-2.0/schemas/org.mate.NotificationDaemon.gschema.override
+/usr/share/glib-2.0/schemas/20_org.gnome.settings-daemon.plugins.updates.qubes.gschema.override
+/usr/share/glib-2.0/schemas/20_org.gnome.nautilus.qubes.gschema.override
+/usr/share/glib-2.0/schemas/20_org.mate.NotificationDaemon.qubes.gschema.override
 /usr/share/nautilus-python/extensions/qvm_copy_nautilus.py*
 /usr/share/nautilus-python/extensions/qvm_move_nautilus.py*
 /usr/share/nautilus-python/extensions/qvm_dvm_nautilus.py*
 
+%dir %{python3_sitelib}/qubesagent-*.egg-info
+%{python3_sitelib}/qubesagent-*.egg-info/*
+%dir %{python3_sitelib}/qubesagent
+%dir %{python3_sitelib}/qubesagent/__pycache__
+%{python3_sitelib}/qubesagent/__pycache__/*
+%{python3_sitelib}/qubesagent/__init__.py
+%{python3_sitelib}/qubesagent/firewall.py
+%{python3_sitelib}/qubesagent/test_firewall.py
+
 /usr/share/qubes/mime-override/globs
 /usr/share/qubes/qubes-master-key.asc
 %dir /home_volatile
@@ -553,7 +568,6 @@ The Qubes core startup configuration for SysV init (or upstart).
 /etc/init.d/qubes-dvm
 /etc/init.d/qubes-core-netvm
 /etc/init.d/qubes-firewall
-/etc/init.d/qubes-netwatcher
 /etc/init.d/qubes-iptables
 /etc/init.d/qubes-updates-proxy
 /etc/init.d/qubes-qrexec-agent
@@ -588,6 +602,9 @@ for svc in %qubes_services ; do
     fi
 done
 
+# dropped services
+chkconfig qubes-netwatcher off || :
+
 # TODO: make this not display the silly message about security context...
 sed -i s/^id:.:initdefault:/id:3:initdefault:/ /etc/inittab
 
@@ -620,7 +637,6 @@ The Qubes core startup configuration for SystemD init.
 /lib/systemd/system/qubes-misc-post.service
 /lib/systemd/system/qubes-firewall.service
 /lib/systemd/system/qubes-mount-dirs.service
-/lib/systemd/system/qubes-netwatcher.service
 /lib/systemd/system/qubes-network.service
 /lib/systemd/system/qubes-iptables.service
 /lib/systemd/system/qubes-sysinit.service

run-tests

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+: "${PYTHON:=python3}"
+: "${ROOTDIR:=.}"
+: "${TESTPYTHONPATH:=$ROOTDIR/test-packages}"
+
+PYTHONPATH="${TESTPYTHONPATH}:${PYTHONPATH}"
+export PYTHONPATH
+
+[ -r version ] || ln -s ${ROOTDIR}/version ./
+[ -r setup.py ] || ln -s ${ROOTDIR}/setup.py ./
+"${PYTHON}" ./setup.py egg_info --egg-base "${TESTPYTHONPATH}"
+"${PYTHON}" -m coverage run -m unittest discover -p '*.py' -v "$@"

setup.py

@@ -0,0 +1,22 @@
+# vim: fileencoding=utf-8
+
+import setuptools
+
+if __name__ == '__main__':
+    setuptools.setup(
+        name='qubesagent',
+        version=open('version').read().strip(),
+        author='Invisible Things Lab',
+        author_email='marmarek@invisiblethingslab.com',
+        description='Qubes core-agent-linux package',
+        license='GPL2+',
+        url='https://www.qubes-os.org/',
+
+        packages=('qubesagent',),
+
+        entry_points={
+            'console_scripts': [
+                'qubes-firewall = qubesagent.firewall:main'
+            ],
+        }
+    )

vm-init.d/qubes-netwatcher

@@ -1,50 +0,0 @@
-#!/bin/bash
-#
-# chkconfig: 345 93 93
-# description: Starts Qubes Network monitor
-#
-# Source function library.
-. /etc/rc.d/init.d/functions
-
-# Source Qubes library.
-. /usr/lib/qubes/init/functions
-
-PIDFILE=/var/run/qubes/qubes-netwatcher.pid
-
-start()
-{
-    have_qubesdb || return
-
-    if qsvc qubes-netwatcher ; then
-        echo -n $"Starting Qubes Network monitor:"
-        /sbin/ethtool -K eth0 sg off
-        /usr/sbin/qubes-netwatcher &
-        success
-        echo ""
-    fi
-}
-
-stop()
-{
-    if [ -r "$PIDFILE" ]; then
-        echo -n "Stopping Qubes Network monitor:"
-        kill -9 $(cat $PIDFILE) 2>/dev/null  && success || failure
-        echo ""
-    fi
-	return 0
-}
-
-case "$1" in
-  start)
-	start
-	;;
-  stop)
-	stop
-	;;
-  *)
-	echo $"Usage: $0 {start|stop}"
-	exit 3
-	;;
-esac
-
-exit $RETVAL

vm-systemd/75-qubes-vm.preset

@@ -88,7 +88,6 @@ enable qubes-network.service
 enable qubes-qrexec-agent.service
 enable qubes-mount-dirs.service
 enable qubes-firewall.service
-enable qubes-netwatcher.service
 enable qubes-meminfo-writer.service
 enable qubes-iptables.service
 enable haveged.service

vm-systemd/qubes-netwatcher.service

@@ -1,10 +0,0 @@
-[Unit]
-Description=Qubes network monitor
-ConditionPathExists=/var/run/qubes-service/qubes-netwatcher
-After=network-pre.target qubes-firewall.service
-
-[Service]
-ExecStart=/usr/sbin/qubes-netwatcher
-
-[Install]
-WantedBy=multi-user.target

vm-systemd/qubes-sysinit.sh

@@ -5,10 +5,20 @@
 
 # List of services enabled by default (in case of absence of qubesdb entry)
 DEFAULT_ENABLED_NETVM="network-manager qubes-network qubes-update-check qubes-updates-proxy"
-DEFAULT_ENABLED_PROXYVM="meminfo-writer qubes-network qubes-firewall qubes-netwatcher qubes-update-check"
-DEFAULT_ENABLED_APPVM="meminfo-writer cups qubes-update-check"
+DEFAULT_ENABLED_PROXYVM="qubes-network qubes-firewall qubes-update-check"
+DEFAULT_ENABLED_APPVM="cups qubes-update-check"
 DEFAULT_ENABLED_TEMPLATEVM="$DEFAULT_ENABLED_APPVM updates-proxy-setup"
-DEFAULT_ENABLED="meminfo-writer"
+DEFAULT_ENABLED=""
+
+if [ -z "`ls /sys/bus/pci/devices/`" ]; then
+    # do not enable meminfo-writer (so qmemman for this domain) when any PCI
+    # device is present
+    DEFAULT_ENABLED="$DEFAULT_ENABLED meminfo-writer"
+    DEFAULT_ENABLED_APPVM="$DEFAULT_ENABLED_APPVM meminfo-writer"
+    DEFAULT_ENABLED_PROXYVM="$DEFAULT_ENABLED_PROXYVM meminfo-writer"
+    DEFAULT_ENABLED_TEMPLATEVM="$DEFAULT_ENABLED_TEMPLATEVM meminfo-writer"
+fi
+
 
 if systemd_version_changed ; then
     # Ensure we're running right version of systemd (the one started by initrd may be different)