From 68b61c2c6d1b18dff4db1c06f6c82521202b7a2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 19 Dec 2019 01:14:39 +0100
Subject: [PATCH 01/25] network: setup anti-spoofing firewall rules before
 enabling the interface

Previously enabling the interface was the first action in the setup
steps. Linux theoretically do not forward the traffic until proper
IP address and route is added to the interface (depending on rp_filter
setting). But instead of relying on this opaque behavior better setup
anti-spoofing rules earlier. Also, add 'set -o pipefail' for more
reliable error handling.
Note the rules for actual VM traffic (qvm-firewall) are properly
enforced - until those rules are loaded, traffic from appropriate vif
interface is blocked. But this relies on proper source IP address,
anti-spoofing rules need to be setup race-free.

Reported-by: Demi M. Obenour <demiobenour@gmail.com>
---
 network/vif-route-qubes | 38 +++++++++++++++++++++++++-------------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/network/vif-route-qubes b/network/vif-route-qubes
index 724d22a..8c1426f 100755
--- a/network/vif-route-qubes
+++ b/network/vif-route-qubes
@@ -24,6 +24,7 @@ dir=$(dirname "$0")
 # shellcheck disable=SC1091,SC1090
 . "$dir/vif-common.sh"
 
+set -o pipefail
 #main_ip=$(dom0_ip)
 
 # Network Hooks for triggering supplementary actions on AppVM connect
@@ -95,7 +96,6 @@ fi
 # shellcheck disable=SC2154
 case "$command" in
     online)
-        ifconfig "${vif}" up
         echo 1 >"/proc/sys/net/ipv4/conf/${vif}/proxy_arp"
         ipcmd='add'
         iptables_cmd='-I PREROUTING 1'
@@ -118,6 +118,30 @@ domid=${domid/.*/}
 #  32752 is max XID aka domid
 metric=$(( 32752 - domid ))
 
+# add anti-spoofing rules before enabling the interface
+if [ "${ip}" ]; then
+    # If we've been given a list of IP addresses, then add routes from dom0 to
+    # the guest using those addresses.
+    for addr in ${ip};
+    do
+        if [[ "$addr" = *:* ]]; then
+            ipt=ip6tables-restore
+        else
+            ipt=iptables-restore
+        fi
+        echo -e "*raw\\n$iptables_cmd -i ${vif} ! -s ${addr} -j DROP\\nCOMMIT" | \
+            ${cmdprefix} $ipt --noflush $ipt_arg
+    done
+    # if no IPv6 is assigned, block all IPv6 traffic on that interface
+    if ! [[ "$ip" = *:* ]]; then
+        echo -e "*raw\\n$iptables_cmd -i ${vif} -j DROP\\nCOMMIT" | \
+            ${cmdprefix} ip6tables-restore --noflush $ipt_arg
+    fi
+fi
+
+if [ "$command" = "online" ]; then
+    ifconfig "${vif}" up
+fi
 
 if [ "${ip}" ]; then
     # If we've been given a list of IP addresses, then add routes from dom0 to
@@ -129,21 +153,9 @@ if [ "${ip}" ]; then
             continue
         fi
         ${cmdprefix} ip route "${ipcmd}" "${addr}" dev "${vif}" metric "$metric"
-        if [[ "$addr" = *:* ]]; then
-            ipt=ip6tables-restore
-        else
-            ipt=iptables-restore
-        fi
-        echo -e "*raw\\n$iptables_cmd -i ${vif} ! -s ${addr} -j DROP\\nCOMMIT" | \
-            ${cmdprefix} $ipt --noflush $ipt_arg
 
         network_hooks "${command}" "${vif}" "${addr}"
     done
-    # if no IPv6 is assigned, block all IPv6 traffic on that interface
-    if ! [[ "$ip" = *:* ]]; then
-        echo -e "*raw\\n$iptables_cmd -i ${vif} -j DROP\\nCOMMIT" | \
-            ${cmdprefix} ip6tables-restore --noflush $ipt_arg
-    fi
     ${cmdprefix} ip addr "${ipcmd}" "${back_ip}/32" dev "${vif}"
     if [ "${back_ip6}" ] && [[ "${back_ip6}" != "fe80:"* ]] && [[ "$ipv6_disabled" != 1 ]]; then
         ${cmdprefix} ip addr "${ipcmd}" "${back_ip6}/128" dev "${vif}"

From 74f5fb5ac7160f145e8383d39d7c41c240f1f0ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 18 Dec 2019 05:57:45 +0100
Subject: [PATCH 02/25] network: prevent IP spoofing on upstream (eth0)
 interface

Currently there is just one anti-spoofing firewall rule ensuring packets
coming through vif+ interfaces have the right source address. Add
another rule ensuring that addresses that belongs to VMs behind those
vif+ interface do not appear on other interfaces (specifically eth0, but
also physical ones).

Normally it wouldn't be an issue because of rp_filter (doing the same
based on route table), default DROP in FORWARD chain and also conntrack
(the need to guess exact port numbers and sequence numbers). But it
appears all three mechanisms are ineffective in some cases:
 - rp_filter in many distributions (including Fedora and Debian) was
 switched to Loose Mode, which doesn't verify exact interface
 - there is a rule in FORWARD table allowing established connections and
 conntrack does not keep track of input/output interfaces
 - CVE-2019-14899 allows to guess all the data needed to inject packets

Reported-by: Demi M. Obenour <demiobenour@gmail.com>
---
 network/vif-route-qubes | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/network/vif-route-qubes b/network/vif-route-qubes
index 8c1426f..345b120 100755
--- a/network/vif-route-qubes
+++ b/network/vif-route-qubes
@@ -129,7 +129,10 @@ if [ "${ip}" ]; then
         else
             ipt=iptables-restore
         fi
-        echo -e "*raw\\n$iptables_cmd -i ${vif} ! -s ${addr} -j DROP\\nCOMMIT" | \
+        printf '%s\n' "*raw" \
+            "$iptables_cmd -i ${vif} ! -s ${addr} -j DROP" \
+            "$iptables_cmd ! -i vif+ -s ${addr} -j DROP" \
+            "COMMIT" | \
             ${cmdprefix} $ipt --noflush $ipt_arg
     done
     # if no IPv6 is assigned, block all IPv6 traffic on that interface

From aa71677cbd881516b8e61f5e7a8c3a80a0dbbe50 Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <athena@invisiblethingslab.com>
Date: Tue, 10 Nov 2020 16:28:53 -0500
Subject: [PATCH 03/25] Add permanent neighbor entries

This allows network traffic to flow even if ARP and NDP do not work or
ave explicitly been disabled.
---
 network/setup-ip        | 18 +++++++++++++++---
 network/vif-route-qubes |  6 +++++-
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/network/setup-ip b/network/setup-ip
index 917b028..b8900d0 100755
--- a/network/setup-ip
+++ b/network/setup-ip
@@ -4,6 +4,14 @@
 # shellcheck disable=SC1091
 . /usr/lib/qubes/init/functions
 
+add_host_route () {
+    /sbin/ip -- route replace to unicast "$1" dev "$2" onlink scope host
+}
+
+add_default_route () {
+    /sbin/ip -- route replace to unicast default via "$1" dev "$2" onlink
+}
+
 configure_network() {
     local MAC="$1"
     local INTERFACE="$2"
@@ -17,20 +25,24 @@ configure_network() {
     local secondary_dns="${10}"
 
     /sbin/ifconfig "$INTERFACE" "$ip" netmask "$netmask"
+    /sbin/ip -- neighbour replace to "$ip" dev "$INTERFACE" \
+        lladdr "$MAC" nud permanent
     if [ -n "$ip6" ]; then
         /sbin/ifconfig "$INTERFACE" add "$ip6/$netmask6"
+        /sbin/ip -- neighbour replace to "$ip6" dev "$INTERFACE" \
+            lladdr "$MAC" nud permanent
     fi
     /sbin/ifconfig "$INTERFACE" up
 
     if [ -n "$gateway" ]; then
         /sbin/route add -host "$gateway" dev "$INTERFACE"
         if [ -n "$gateway6" ] && ! echo "$gateway6" | grep -q "^fe80:"; then
-            /sbin/route -6 add "$gateway6/$netmask6" dev "$INTERFACE"
+            add_route "$gateway6/$netmask6" "$INTERFACE"
         fi
         if ! qsvc disable-default-route ; then
-            /sbin/route add default gw "$gateway"
+            add_default_route "$gateway" "$INTERFACE"
             if [ -n "$gateway6" ]; then
-                /sbin/route -6 add default gw "$gateway6" dev "$INTERFACE"
+                add_default_route "$gateway6" "$INTERFACE"
             fi
         fi
     fi
diff --git a/network/vif-route-qubes b/network/vif-route-qubes
index 345b120..82c0549 100755
--- a/network/vif-route-qubes
+++ b/network/vif-route-qubes
@@ -134,6 +134,10 @@ if [ "${ip}" ]; then
             "$iptables_cmd ! -i vif+ -s ${addr} -j DROP" \
             "COMMIT" | \
             ${cmdprefix} $ipt --noflush $ipt_arg
+        if [[ "$command" = 'online' ]]; then
+            ip neighbour "${ipcmd}" to "${addr}" \
+                dev "${vif}" lladdr 00:16:3e:5e:6c:6c:00 nud permanent
+        fi
     done
     # if no IPv6 is assigned, block all IPv6 traffic on that interface
     if ! [[ "$ip" = *:* ]]; then
@@ -155,7 +159,7 @@ if [ "${ip}" ]; then
             log error "Cannot set IPv6 route to ${addr}, IPv6 disabled in the kernel"
             continue
         fi
-        ${cmdprefix} ip route "${ipcmd}" "${addr}" dev "${vif}" metric "$metric"
+        ${cmdprefix} ip route "${ipcmd}" "${addr}" dev "${vif}" metric "$metric" onlink
 
         network_hooks "${command}" "${vif}" "${addr}"
     done

From 0a322958e4cc399cfa15382aa6aa5b713dbb3cb9 Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <athena@invisiblethingslab.com>
Date: Tue, 10 Nov 2020 22:09:54 -0500
Subject: [PATCH 04/25] =?UTF-8?q?Add=20gateway=20IP+MAC,=20not=20VM?=
 =?UTF-8?q?=E2=80=99s=20own?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 network/setup-ip | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/network/setup-ip b/network/setup-ip
index b8900d0..b050d63 100755
--- a/network/setup-ip
+++ b/network/setup-ip
@@ -23,21 +23,22 @@ configure_network() {
     local gateway6="$8"
     local primary_dns="$9"
     local secondary_dns="${10}"
+    local netvm_mac=fe:ff:ff:ff:ff:ff
 
     /sbin/ifconfig "$INTERFACE" "$ip" netmask "$netmask"
-    /sbin/ip -- neighbour replace to "$ip" dev "$INTERFACE" \
-        lladdr "$MAC" nud permanent
+    /sbin/ip -- neighbour replace to "$gateway" dev "$INTERFACE" \
+        lladdr "$netvm_mac" nud permanent
     if [ -n "$ip6" ]; then
         /sbin/ifconfig "$INTERFACE" add "$ip6/$netmask6"
-        /sbin/ip -- neighbour replace to "$ip6" dev "$INTERFACE" \
-            lladdr "$MAC" nud permanent
+        /sbin/ip -- neighbour replace to "$gateway6" dev "$INTERFACE" \
+            lladdr "$netvm_mac" nud permanent
     fi
     /sbin/ifconfig "$INTERFACE" up
 
     if [ -n "$gateway" ]; then
-        /sbin/route add -host "$gateway" dev "$INTERFACE"
+        add_host_route "$gateway" "$INTERFACE"
         if [ -n "$gateway6" ] && ! echo "$gateway6" | grep -q "^fe80:"; then
-            add_route "$gateway6/$netmask6" "$INTERFACE"
+            add_host_route "$gateway6/$netmask6" "$INTERFACE"
         fi
         if ! qsvc disable-default-route ; then
             add_default_route "$gateway" "$INTERFACE"

From 377add43d19c0ac93d9e060675f6de4ce1eb42c5 Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <athena@invisiblethingslab.com>
Date: Tue, 10 Nov 2020 22:31:18 -0500
Subject: [PATCH 05/25] =?UTF-8?q?Don=E2=80=99t=20hardcode=20MAC=20addresse?=
 =?UTF-8?q?s?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 network/vif-route-qubes | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/network/vif-route-qubes b/network/vif-route-qubes
index 82c0549..ec33462 100755
--- a/network/vif-route-qubes
+++ b/network/vif-route-qubes
@@ -112,8 +112,8 @@ case "$command" in
         ;;
 esac
 
-domid=${vif/vif/}
-domid=${domid/.*/}
+[[ $vif =~ ^vif([0-9]+)\.([0-9]+)$ ]] || exit 1
+domid=${BASH_REMATCH[1]} sub=${BASH_REMATCH[2]}
 # metric must be possitive, but prefer later interface
 #  32752 is max XID aka domid
 metric=$(( 32752 - domid ))
@@ -135,8 +135,9 @@ if [ "${ip}" ]; then
             "COMMIT" | \
             ${cmdprefix} $ipt --noflush $ipt_arg
         if [[ "$command" = 'online' ]]; then
-            ip neighbour "${ipcmd}" to "${addr}" \
-                dev "${vif}" lladdr 00:16:3e:5e:6c:6c:00 nud permanent
+            mac=$(xenstore-read "backend/vif/$domid/$sub/mac") &&
+            ip -- neighbour "${ipcmd}" to "${addr}" \
+                dev "${vif}" lladdr "$mac" nud permanent
         fi
     done
     # if no IPv6 is assigned, block all IPv6 traffic on that interface

From 3e7552856fe78bae1070ab3f35ee82514c0d2359 Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <athena@invisiblethingslab.com>
Date: Wed, 11 Nov 2020 14:00:24 -0500
Subject: [PATCH 06/25] Fix running under -euo pipefail

Some qubesdb-read commands are expected to fail.  I ultimately did not
wind up including -e, but this version should be ready for it.
---
 network/setup-ip | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/network/setup-ip b/network/setup-ip
index b050d63..35e9fba 100755
--- a/network/setup-ip
+++ b/network/setup-ip
@@ -3,6 +3,7 @@
 # Source Qubes library.
 # shellcheck disable=SC1091
 . /usr/lib/qubes/init/functions
+set -uo pipefail
 
 add_host_route () {
     /sbin/ip -- route replace to unicast "$1" dev "$2" onlink scope host
@@ -176,27 +177,27 @@ if [ -n "$INTERFACE" ]; then
     if [ "$ACTION" == "add" ]; then
         MAC="$(get_mac_from_iface "$INTERFACE")"
         if [ -n "$MAC" ]; then
-            ip="$(/usr/bin/qubesdb-read "/net-config/$MAC/ip" 2> /dev/null)"
-            ip6="$(/usr/bin/qubesdb-read "/net-config/$MAC/ip6" 2> /dev/null)"
-            netmask="$(/usr/bin/qubesdb-read "/net-config/$MAC/netmask" 2> /dev/null)"
-            netmask6="$(/usr/bin/qubesdb-read "/net-config/$MAC/netmask6" 2> /dev/null)"
-            gateway="$(/usr/bin/qubesdb-read "/net-config/$MAC/gateway" 2> /dev/null)"
-            gateway6="$(/usr/bin/qubesdb-read "/net-config/$MAC/gateway6" 2> /dev/null)"
+            ip="$(/usr/bin/qubesdb-read "/net-config/$MAC/ip" 2> /dev/null)" || ip=
+            ip6="$(/usr/bin/qubesdb-read "/net-config/$MAC/ip6" 2> /dev/null)" || ip6=
+            netmask="$(/usr/bin/qubesdb-read "/net-config/$MAC/netmask" 2> /dev/null)" || netmask=
+            netmask6="$(/usr/bin/qubesdb-read "/net-config/$MAC/netmask6" 2> /dev/null)" || netmask6=
+            gateway="$(/usr/bin/qubesdb-read "/net-config/$MAC/gateway" 2> /dev/null)" || gateway=
+            gateway6="$(/usr/bin/qubesdb-read "/net-config/$MAC/gateway6" 2> /dev/null)" || gateway6=
 
             # Handle legacy values
-            LEGACY_MAC="$(/usr/bin/qubesdb-read /qubes-mac 2> /dev/null)"
+            LEGACY_MAC="$(/usr/bin/qubesdb-read /qubes-mac 2> /dev/null)" || LEGACY_MAC=
             if [ "$MAC" == "$LEGACY_MAC" ] || [ -z "$LEGACY_MAC" ]; then
                 if [ -z "$ip" ]; then
-                    ip="$(/usr/bin/qubesdb-read /qubes-ip 2> /dev/null)"
+                    ip="$(/usr/bin/qubesdb-read /qubes-ip 2> /dev/null)" || ip=
                 fi
                 if [ -z "$ip6" ]; then
-                    ip6="$(/usr/bin/qubesdb-read /qubes-ip6 2> /dev/null)"
+                    ip6="$(/usr/bin/qubesdb-read /qubes-ip6 2> /dev/null)" || ip6=
                 fi
                 if [ -z "$gateway" ]; then
-                    gateway="$(/usr/bin/qubesdb-read /qubes-gateway 2> /dev/null)"
+                    gateway="$(/usr/bin/qubesdb-read /qubes-gateway 2> /dev/null)" || gateway=
                 fi
                 if [ -z "$gateway6" ]; then
-                    gateway6="$(/usr/bin/qubesdb-read /qubes-gateway6 2> /dev/null)"
+                    gateway6="$(/usr/bin/qubesdb-read /qubes-gateway6 2> /dev/null)" || gateway6=
                 fi
             fi
 
@@ -207,8 +208,8 @@ if [ -n "$INTERFACE" ]; then
                 netmask6="128"
             fi
 
-            primary_dns=$(/usr/bin/qubesdb-read /qubes-primary-dns 2>/dev/null)
-            secondary_dns=$(/usr/bin/qubesdb-read /qubes-secondary-dns 2>/dev/null)
+            primary_dns=$(/usr/bin/qubesdb-read /qubes-primary-dns 2>/dev/null) || primary_dns=
+            secondary_dns=$(/usr/bin/qubesdb-read /qubes-secondary-dns 2>/dev/null) || secondary_dns=
 
             if [ -n "$ip" ]; then
                 /sbin/ethtool -K "$INTERFACE" sg off
@@ -221,7 +222,7 @@ if [ -n "$INTERFACE" ]; then
                     configure_network "$MAC" "$INTERFACE" "$ip" "$ip6" "$netmask" "$netmask6" "$gateway" "$gateway6" "$primary_dns" "$secondary_dns"
                 fi
 
-                network=$(qubesdb-read /qubes-netvm-network 2>/dev/null)
+                network=$(qubesdb-read /qubes-netvm-network 2>/dev/null) || network=
                 if [ -n "$network" ]; then
                     if ! qsvc disable-dns-server; then
                         configure_qubes_ns

From 9646acb18e6eaa2e29e8398833c058d79c7d1167 Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <athena@invisiblethingslab.com>
Date: Fri, 13 Nov 2020 12:51:15 -0500
Subject: [PATCH 07/25] =?UTF-8?q?Don=E2=80=99t=20use=20onlink=20flag=20for?=
 =?UTF-8?q?=20nexthop?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is rejected by the kernel.
---
 network/vif-route-qubes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/vif-route-qubes b/network/vif-route-qubes
index ec33462..e87806c 100755
--- a/network/vif-route-qubes
+++ b/network/vif-route-qubes
@@ -160,7 +160,7 @@ if [ "${ip}" ]; then
             log error "Cannot set IPv6 route to ${addr}, IPv6 disabled in the kernel"
             continue
         fi
-        ${cmdprefix} ip route "${ipcmd}" "${addr}" dev "${vif}" metric "$metric" onlink
+        ${cmdprefix} ip route "${ipcmd}" "${addr}" dev "${vif}" metric "$metric"
 
         network_hooks "${command}" "${vif}" "${addr}"
     done

From 791b08c2ecf6021946531c38fa228f0926c86ac4 Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <athena@invisiblethingslab.com>
Date: Fri, 13 Nov 2020 13:15:24 -0500
Subject: [PATCH 08/25] vif-route-qubes: better input validation

The input is trusted, but this will help debugging if something goes
wrong.
---
 network/vif-route-qubes | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/network/vif-route-qubes b/network/vif-route-qubes
index e87806c..85881b7 100755
--- a/network/vif-route-qubes
+++ b/network/vif-route-qubes
@@ -112,16 +112,28 @@ case "$command" in
         ;;
 esac
 
-[[ $vif =~ ^vif([0-9]+)\.([0-9]+)$ ]] || exit 1
+readonly max_domid=32752
+
+# if domid is 0 something is seriously wrong, so don’t check for that case
+if ! [[ $vif =~ ^vif([1-9][0-9]{,4})\.(0|[1-9][0-9]*)$ ]]; then
+    printf 'Bad interface name %q\n' "$vif">&2
+    exit 1
+fi
+
 domid=${BASH_REMATCH[1]} sub=${BASH_REMATCH[2]}
-# metric must be possitive, but prefer later interface
+
+# metric must be positive, but prefer later interface
 #  32752 is max XID aka domid
-metric=$(( 32752 - domid ))
+if (( domid > max_domid )); then
+    printf %s\\n "domid $domid too large"
+    exit 1
+fi
+metric=$(( max_domid - domid ))
 
 # add anti-spoofing rules before enabling the interface
 if [ "${ip}" ]; then
-    # If we've been given a list of IP addresses, then add routes from dom0 to
-    # the guest using those addresses.
+    # If we’ve been given a list of IP addresses, then add routes from us to
+    # the VMs we serve using those addresses.
     for addr in ${ip};
     do
         if [[ "$addr" = *:* ]]; then

From 6517cca2a4f56568812f53b9a93784b16d894012 Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <athena@invisiblethingslab.com>
Date: Thu, 19 Nov 2020 12:04:02 -0500
Subject: [PATCH 09/25] NAT network namespaces need neighbor entries

If we are using a NAT network namespace, it needs its own neighbor
entries.  For consistency, give it the same MAC address as the VM it
connects to.
---
 network/vif-qubes-nat.sh | 10 +++++--
 network/vif-route-qubes  | 65 +++++++++++++++++++---------------------
 2 files changed, 37 insertions(+), 38 deletions(-)

diff --git a/network/vif-qubes-nat.sh b/network/vif-qubes-nat.sh
index 9833e45..2789d34 100755
--- a/network/vif-qubes-nat.sh
+++ b/network/vif-qubes-nat.sh
@@ -46,9 +46,12 @@ if test "$command" == online; then
     run ip link set "$netns_appvm_if" netns "$netns"
 
     # keep the same MAC as the real vif interface, so NetworkManager will still
-    # ignore it
-    run ip link add "$netns_netvm_if" type veth peer name "$netvm_if" address fe:ff:ff:ff:ff:ff
-    run ip link set "$netns_netvm_if" netns "$netns"
+    # ignore it.
+    # for the peer interface, make sure that it has the same MAC address
+    # as the actual VM, so that our neighbor entry works.
+    run ip link add name "$netns_netvm_if" address "$mac" type veth \
+        peer name "$netvm_if" address fe:ff:ff:ff:ff:ff
+    run ip link set dev "$netns_netvm_if" netns "$netns"
 
     netns ip6tables -t raw -I PREROUTING -j DROP
     netns ip6tables -P INPUT DROP
@@ -85,6 +88,7 @@ if test "$command" == online; then
         netns iptables -t nat -I POSTROUTING -o "$netns_appvm_if" -s "$netvm_dns2_ip" -j SNAT --to-source "$appvm_dns2_ip"
     fi
 
+    netns ip neighbour add to "$appvm_ip" dev "$netns_appvm_if" lladdr "$mac" nud permanent
     netns ip addr add "$netvm_ip" dev "$netns_netvm_if"
     netns ip addr add "$appvm_gw_ip" dev "$netns_appvm_if"
 
diff --git a/network/vif-route-qubes b/network/vif-route-qubes
index 85881b7..1cf43d2 100755
--- a/network/vif-route-qubes
+++ b/network/vif-route-qubes
@@ -78,40 +78,6 @@ if [ "${ip}" ]; then
     appvm_ip="$(qubesdb-read "/mapped-ip/$ip4/visible-ip" 2>/dev/null || :)"
 fi
 
-# Apply NAT if IP visible from the VM is different than the "real" one
-# See vif-qubes-nat.sh for details
-# XXX: supported only for the first IPv4 address, IPv6 is dropped if this
-# feature is enabled
-if [ -n "$appvm_ip" ] && [ -n "$appvm_gw_ip" ] && [ "$appvm_ip" != "$netvm_ip" ]; then
-    # shellcheck disable=SC2154
-    if test "$command" == online; then
-        # shellcheck disable=SC2154
-        echo 1 >"/proc/sys/net/ipv4/conf/${vif}/proxy_arp"
-    fi
-
-    # shellcheck source=network/vif-qubes-nat.sh
-    . "$dir/vif-qubes-nat.sh"
-fi
-
-# shellcheck disable=SC2154
-case "$command" in
-    online)
-        echo 1 >"/proc/sys/net/ipv4/conf/${vif}/proxy_arp"
-        ipcmd='add'
-        iptables_cmd='-I PREROUTING 1'
-        cmdprefix=''
-        ipv6_disabled=$(cat /proc/sys/net/ipv6/conf/"${vif}"/disable_ipv6 || echo 1)
-        ;;
-    offline)
-        do_without_error ifdown "${vif}"
-        ipcmd='del'
-        iptables_cmd='-D PREROUTING'
-        cmdprefix='do_without_error'
-        # cleanup IPv6 config even if _now_ it is disabled
-        ipv6_disabled=0
-        ;;
-esac
-
 readonly max_domid=32752
 
 # if domid is 0 something is seriously wrong, so don’t check for that case
@@ -130,6 +96,36 @@ if (( domid > max_domid )); then
 fi
 metric=$(( max_domid - domid ))
 
+# shellcheck disable=SC2154
+case "$command" in
+    online)
+        echo 1 >"/proc/sys/net/ipv4/conf/${vif}/proxy_arp"
+        ipcmd='add'
+        iptables_cmd='-I PREROUTING 1'
+        cmdprefix=''
+        ipv6_disabled=$(cat /proc/sys/net/ipv6/conf/"${vif}"/disable_ipv6 || echo 1)
+        # without a MAC address we will fail later with a confusing error
+        mac=$(xenstore-read "backend/vif/$domid/$sub/mac") || exit 1
+        ;;
+    offline)
+        do_without_error ifdown "${vif}"
+        ipcmd='del'
+        iptables_cmd='-D PREROUTING'
+        cmdprefix='do_without_error'
+        # cleanup IPv6 config even if _now_ it is disabled
+        ipv6_disabled=0
+        ;;
+esac
+
+# Apply NAT if IP visible from the VM is different than the "real" one
+# See vif-qubes-nat.sh for details
+# XXX: supported only for the first IPv4 address, IPv6 is dropped if this
+# feature is enabled
+if [ -n "$appvm_ip" ] && [ -n "$appvm_gw_ip" ] && [ "$appvm_ip" != "$netvm_ip" ]; then
+    # shellcheck source=network/vif-qubes-nat.sh
+    . "$dir/vif-qubes-nat.sh"
+fi
+
 # add anti-spoofing rules before enabling the interface
 if [ "${ip}" ]; then
     # If we’ve been given a list of IP addresses, then add routes from us to
@@ -147,7 +143,6 @@ if [ "${ip}" ]; then
             "COMMIT" | \
             ${cmdprefix} $ipt --noflush $ipt_arg
         if [[ "$command" = 'online' ]]; then
-            mac=$(xenstore-read "backend/vif/$domid/$sub/mac") &&
             ip -- neighbour "${ipcmd}" to "${addr}" \
                 dev "${vif}" lladdr "$mac" nud permanent
         fi

From 097342bd08a3fd2004038436d4f07e68c0af223d Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <athena@invisiblethingslab.com>
Date: Thu, 19 Nov 2020 12:10:26 -0500
Subject: [PATCH 10/25] Optimization: use `ip -n` over `ip netns exec`

This saves an exec call.
---
 network/vif-qubes-nat.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/network/vif-qubes-nat.sh b/network/vif-qubes-nat.sh
index 2789d34..672f407 100755
--- a/network/vif-qubes-nat.sh
+++ b/network/vif-qubes-nat.sh
@@ -35,7 +35,12 @@ function run
 
 function netns
 {
-    run ip netns exec "$netns" "$@"
+    if [[ "$1" = 'ip' ]]; then
+        shift
+        run ip -n "$netns" "$@"
+    else
+        run ip netns exec "$netns" "$@"
+    fi
 }
 
 run ip addr flush dev "$netns_appvm_if"

From e4eeb2ee1bc8e69e25f7333961ff4da64b511f2b Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <athena@invisiblethingslab.com>
Date: Thu, 19 Nov 2020 12:16:15 -0500
Subject: [PATCH 11/25] Add NetVM-facing neighbor entry in NAT namespace

Since AppVMs will have their own NetVM-facing neighbor entries, a user
might (correctly) conclude that NetVMs do not need ARP or NDP enabled.
For this to work with NAT namespaces, they need their own neighbor
entries.
---
 network/vif-qubes-nat.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/network/vif-qubes-nat.sh b/network/vif-qubes-nat.sh
index 672f407..058d8a3 100755
--- a/network/vif-qubes-nat.sh
+++ b/network/vif-qubes-nat.sh
@@ -26,6 +26,7 @@ netns_appvm_if="${vif}"
 #               '----------------------------------'
 #
 
+readonly netvm_mac=fe:ff:ff:ff:ff:ff
 
 function run
 {
@@ -55,7 +56,7 @@ if test "$command" == online; then
     # for the peer interface, make sure that it has the same MAC address
     # as the actual VM, so that our neighbor entry works.
     run ip link add name "$netns_netvm_if" address "$mac" type veth \
-        peer name "$netvm_if" address fe:ff:ff:ff:ff:ff
+        peer name "$netvm_if" address "$netvm_mac"
     run ip link set dev "$netns_netvm_if" netns "$netns"
 
     netns ip6tables -t raw -I PREROUTING -j DROP
@@ -94,6 +95,7 @@ if test "$command" == online; then
     fi
 
     netns ip neighbour add to "$appvm_ip" dev "$netns_appvm_if" lladdr "$mac" nud permanent
+    netns ip neighbour add to "$netvm_ip" dev "$netns_netvm_if" lladdr "$netvm_mac" nud permanent
     netns ip addr add "$netvm_ip" dev "$netns_netvm_if"
     netns ip addr add "$appvm_gw_ip" dev "$netns_appvm_if"
 

From 9d10ecc08f8903aa94b76b644038f155a9453c7f Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <athena@invisiblethingslab.com>
Date: Thu, 19 Nov 2020 15:19:40 -0500
Subject: [PATCH 12/25] Remove commented-out code

---
 network/vif-qubes-nat.sh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/network/vif-qubes-nat.sh b/network/vif-qubes-nat.sh
index 058d8a3..21b3bb1 100755
--- a/network/vif-qubes-nat.sh
+++ b/network/vif-qubes-nat.sh
@@ -105,10 +105,4 @@ if test "$command" == online; then
     netns ip route add "$appvm_ip" dev "$netns_appvm_if" src "$appvm_gw_ip"
     netns ip route add "$netvm_gw_ip" dev "$netns_netvm_if" src "$netvm_ip"
     netns ip route add default via "$netvm_gw_ip" dev "$netns_netvm_if" src "$netvm_ip"
-
-
-    #run ip addr add "$netvm_gw_ip" dev "$netvm_if"
-    #run ip link set "$netvm_if" up
-    #run ip route add "$netvm_ip" dev "$netvm_if" src "$netvm_gw_ip"
 fi
-

From 0580fe545b7b4616f963694e1eddcc9788b53f38 Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <athena@invisiblethingslab.com>
Date: Sun, 22 Nov 2020 17:52:54 -0500
Subject: [PATCH 13/25] Use netvm_gw_ip instead of netvm_ip

They are usually identical, but this is not guaranteed.
---
 network/vif-qubes-nat.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/vif-qubes-nat.sh b/network/vif-qubes-nat.sh
index 21b3bb1..0c38731 100755
--- a/network/vif-qubes-nat.sh
+++ b/network/vif-qubes-nat.sh
@@ -95,7 +95,7 @@ if test "$command" == online; then
     fi
 
     netns ip neighbour add to "$appvm_ip" dev "$netns_appvm_if" lladdr "$mac" nud permanent
-    netns ip neighbour add to "$netvm_ip" dev "$netns_netvm_if" lladdr "$netvm_mac" nud permanent
+    netns ip neighbour add to "$netvm_gw_ip" dev "$netns_netvm_if" lladdr "$netvm_mac" nud permanent
     netns ip addr add "$netvm_ip" dev "$netns_netvm_if"
     netns ip addr add "$appvm_gw_ip" dev "$netns_appvm_if"
 

From 489fde7cb3f08dd486835aa46235f6a1462cca56 Mon Sep 17 00:00:00 2001
From: ejose19 <8742215+ejose19@users.noreply.github.com>
Date: Thu, 26 Nov 2020 13:55:34 -0300
Subject: [PATCH 14/25] Replace custom script reloading with sourcing
 /etc/profile in qubes.GetAppmenus

---
 qubes-rpc/qubes.GetAppmenus | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/qubes-rpc/qubes.GetAppmenus b/qubes-rpc/qubes.GetAppmenus
index 2422a9b..386918f 100755
--- a/qubes-rpc/qubes.GetAppmenus
+++ b/qubes-rpc/qubes.GetAppmenus
@@ -9,12 +9,8 @@
 # Reload scripts in /etc/profile.d/, in case they register additional
 # directories in XDG_DATA_DIRS and we forgot them
 # (e.g. because we are running under sudo).
-for i in /etc/profile.d/*.sh ; do
-    if [ -r "$i" ]; then
-        # shellcheck disable=SC1090
-        . "$i" >/dev/null
-    fi
-done
+# shellcheck disable=SC1091
+source /etc/profile
 
 if [ -z "$XDG_DATA_HOME" ]; then
     user="$(whoami)"

From 06d84b5198a77ccb90b3229982eeca192dd356f8 Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <athena@invisiblethingslab.com>
Date: Fri, 27 Nov 2020 15:25:29 -0500
Subject: [PATCH 15/25] Only allow known-safe characters in socket paths
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The socket path will be included in a shell command and then as a socat
argument, so only allow a small subset of known-safe characters.  In
practice, this has not been a problem because mktemp doesn’t include
these characters in its output.
---
 qubes-rpc/qubes.ShowInTerminal | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/qubes-rpc/qubes.ShowInTerminal b/qubes-rpc/qubes.ShowInTerminal
index d8cb758..f8b5f95 100644
--- a/qubes-rpc/qubes.ShowInTerminal
+++ b/qubes-rpc/qubes.ShowInTerminal
@@ -1,10 +1,11 @@
 #!/bin/bash --
 
 set -eu
+umask 0077
 tmpdir="$(mktemp -d)"
 
-if [[ $tmpdir =~ \" ]]; then
-    echo 'Error: non admissible character detected in sock path.'
+if ! [[ $tmpdir =~ ^/[/A-Za-z0-9._-]+$ ]]; then
+    echo 'Error: non admissible character detected in sock path.'>&2
     exit 1
 fi
 

From f66a494cc2e5e84a19c4abe2d8f6516265457926 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 12 Nov 2020 00:47:05 +0100
Subject: [PATCH 16/25] Allow DHCPv6 replies on uplink interface, if ipv6 is
 enabled

Fixes QubesOS/qubes-issues#5886
---
 network/ip6tables-enabled | 1 +
 1 file changed, 1 insertion(+)

diff --git a/network/ip6tables-enabled b/network/ip6tables-enabled
index fc5aec1..d2e4a56 100644
--- a/network/ip6tables-enabled
+++ b/network/ip6tables-enabled
@@ -26,6 +26,7 @@ COMMIT
 -A INPUT -m state --state INVALID -j DROP
 -A INPUT -i lo -j ACCEPT
 -A INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A INPUT ! -i vif+ -p udp -s fe80::/64 -d fe80::/64 --dport 546 -j ACCEPT
 -A INPUT -i vif+ -p icmpv6 --icmpv6-type router-advertisement -j DROP
 -A INPUT -i vif+ -p icmpv6 --icmpv6-type redirect -j DROP
 -A INPUT -i vif+ -p icmpv6 -j ACCEPT

From 0caa7fcf75a7521b47a5652df4e3b0ef772ee2e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 12 Nov 2020 00:53:48 +0100
Subject: [PATCH 17/25] network: stop IP forwarding before disabling firewall

Stop IP forwarding when stopping qubes-network service (which initially
enables it). This makes ordering against qubes-firewall safe - firewall
is applied before allowing IP forward and then is removed when IP
forward is already disabled.

Fixes QubesOS/qubes-issues#5599
---
 debian/qubes-core-agent-networking.install | 1 +
 rpm_spec/core-agent.spec.in                | 1 +
 vm-systemd/network-proxy-stop.sh           | 7 +++++++
 vm-systemd/qubes-network.service           | 1 +
 4 files changed, 10 insertions(+)
 create mode 100755 vm-systemd/network-proxy-stop.sh

diff --git a/debian/qubes-core-agent-networking.install b/debian/qubes-core-agent-networking.install
index 7f371ef..8c63f10 100644
--- a/debian/qubes-core-agent-networking.install
+++ b/debian/qubes-core-agent-networking.install
@@ -13,6 +13,7 @@ lib/systemd/system/qubes-iptables.service
 lib/systemd/system/qubes-network.service
 lib/systemd/system/qubes-updates-proxy.service
 usr/lib/qubes/init/network-proxy-setup.sh
+usr/lib/qubes/init/network-proxy-stop.sh
 usr/lib/qubes/init/qubes-iptables
 usr/lib/qubes/iptables-updates-proxy
 usr/lib/qubes/qubes-setup-dnat-to-ns
diff --git a/rpm_spec/core-agent.spec.in b/rpm_spec/core-agent.spec.in
index 0ec54e0..3e663d3 100644
--- a/rpm_spec/core-agent.spec.in
+++ b/rpm_spec/core-agent.spec.in
@@ -799,6 +799,7 @@ rm -f %{name}-%{version}
 /lib/systemd/system/qubes-network.service
 /lib/systemd/system/qubes-updates-proxy.service
 /usr/lib/qubes/init/network-proxy-setup.sh
+/usr/lib/qubes/init/network-proxy-stop.sh
 /usr/lib/qubes/init/qubes-iptables
 /usr/lib/qubes/iptables-updates-proxy
 /usr/lib/qubes/qubes-setup-dnat-to-ns
diff --git a/vm-systemd/network-proxy-stop.sh b/vm-systemd/network-proxy-stop.sh
new file mode 100755
index 0000000..4ef924e
--- /dev/null
+++ b/vm-systemd/network-proxy-stop.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+echo 0 > /proc/sys/net/ipv4/ip_forward
+# disable also IPv6 forwarding, if IPv6 applicable
+if [ -w /proc/sys/net/ipv6/conf/all/forwarding ]; then
+    echo 0 > /proc/sys/net/ipv6/conf/all/forwarding
+fi
diff --git a/vm-systemd/qubes-network.service b/vm-systemd/qubes-network.service
index c5aa410..5281bf1 100644
--- a/vm-systemd/qubes-network.service
+++ b/vm-systemd/qubes-network.service
@@ -8,6 +8,7 @@ After=network-pre.target qubes-iptables.service
 Type=oneshot
 RemainAfterExit=yes
 ExecStart=/usr/lib/qubes/init/network-proxy-setup.sh
+ExecStop=/usr/lib/qubes/init/network-proxy-stop.sh
 
 [Install]
 WantedBy=multi-user.target

From e344dcc4c90bcd5e04ec4f1c9a35c3a25dec8269 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 12 Nov 2020 01:14:10 +0100
Subject: [PATCH 18/25] Order qubes-early-vm-config.service before networking

Fixes QubesOS/qubes-issues#5570
---
 vm-systemd/qubes-early-vm-config.service | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vm-systemd/qubes-early-vm-config.service b/vm-systemd/qubes-early-vm-config.service
index 36601b6..dddbc49 100644
--- a/vm-systemd/qubes-early-vm-config.service
+++ b/vm-systemd/qubes-early-vm-config.service
@@ -1,7 +1,7 @@
 [Unit]
 Description=Early Qubes VM settings
 DefaultDependencies=no
-Before=sysinit.target
+Before=sysinit.target network-pre.target
 After=local-fs.target qubes-db.service
 
 [Service]

From dd8de797e3bb0f60bdfc293228beb384242af352 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 12 Nov 2020 01:37:12 +0100
Subject: [PATCH 19/25] Move network uplink setup to a separate service

Previously, network uplink (eth0) was configured in two places:
 - udev (asynchronously)
 - qubes-misc-post.service - at the very end of the boot process

This caused multiple issues:
1. Depending on udev event processing (non-deterministic), network
   uplink could be enabled too early, for example before setting up
   firewall.
2. Again depending on udev processing, it can be enabled quite late in
   the boot process, after network.target is up and services assume
   network already configured. This for example causes qubes-firewall to
   fail DNS queries.
3. If udev happen try to enable enable networking even earlier, it may
   happend before qubesdb-daemon is started, in which case network setup
   fill fail. For this case, there was network re-setup in
   qubes-misc-post service - much later in the boot.

Fix the above by placing network uplink setup in a dedicated
qubes-network-uplink@${INTERFACE}.service unit ordered after
network-pre.target and pulled in by udev based on vif device existence,
to handle also dynamic network attach/detach.
Then, create qubes-network-uplink.service unit waiting for appropriate
interface-specific unit (if one is expected!) and order it before
network.target.

QubesOS/qubes-issues#5576
---
 debian/qubes-core-agent-networking.install |  3 +++
 network/setup-ip                           | 16 ++++++++++++++++
 network/udev-qubes-network.rules           |  6 +++---
 rpm_spec/core-agent.spec.in                |  3 +++
 vm-systemd/75-qubes-vm.preset              |  1 +
 vm-systemd/misc-post.sh                    |  9 ---------
 vm-systemd/network-uplink-wait.sh          | 16 ++++++++++++++++
 vm-systemd/qubes-network-uplink.service    | 11 +++++++++++
 vm-systemd/qubes-network-uplink@.service   | 11 +++++++++++
 9 files changed, 64 insertions(+), 12 deletions(-)
 create mode 100644 vm-systemd/network-uplink-wait.sh
 create mode 100644 vm-systemd/qubes-network-uplink.service
 create mode 100644 vm-systemd/qubes-network-uplink@.service

diff --git a/debian/qubes-core-agent-networking.install b/debian/qubes-core-agent-networking.install
index 8c63f10..c26f251 100644
--- a/debian/qubes-core-agent-networking.install
+++ b/debian/qubes-core-agent-networking.install
@@ -11,9 +11,12 @@ etc/xen/scripts/vif-route-qubes
 lib/systemd/system/qubes-firewall.service
 lib/systemd/system/qubes-iptables.service
 lib/systemd/system/qubes-network.service
+lib/systemd/system/qubes-network-uplink.service
+lib/systemd/system/qubes-network-uplink@.service
 lib/systemd/system/qubes-updates-proxy.service
 usr/lib/qubes/init/network-proxy-setup.sh
 usr/lib/qubes/init/network-proxy-stop.sh
+usr/lib/qubes/init/network-uplink-wait.sh
 usr/lib/qubes/init/qubes-iptables
 usr/lib/qubes/iptables-updates-proxy
 usr/lib/qubes/qubes-setup-dnat-to-ns
diff --git a/network/setup-ip b/network/setup-ip
index 35e9fba..fd2db18 100755
--- a/network/setup-ip
+++ b/network/setup-ip
@@ -173,6 +173,14 @@ qubes_ip_change_hook() {
 
 have_qubesdb || exit 0
 
+ACTION="$1"
+INTERFACE="$2"
+
+if [ -z "$INTERFACE" ]; then
+    echo "Missing INTERFACE argument" >&2
+    exit 1
+fi
+
 if [ -n "$INTERFACE" ]; then
     if [ "$ACTION" == "add" ]; then
         MAC="$(get_mac_from_iface "$INTERFACE")"
@@ -232,8 +240,16 @@ if [ -n "$INTERFACE" ]; then
             fi
         fi
     elif [ "$ACTION" == "remove" ]; then
+        # make sure network is disabled, especially on shutdown, to prevent
+        # leaks when firewall will get stopped too
+        ip link set "$INTERFACE" down 2>/dev/null || :
+
         # If exists, we delete NetworkManager configuration file to prevent duplicate entries
         nm_config="/etc/NetworkManager/system-connections/qubes-uplink-$INTERFACE"
         rm -rf "$nm_config"
+    else
+        echo "Invalid action '$ACTION'" >&2
+        exit 1
     fi
+
 fi
diff --git a/network/udev-qubes-network.rules b/network/udev-qubes-network.rules
index 0ae83b2..fde5fe9 100644
--- a/network/udev-qubes-network.rules
+++ b/network/udev-qubes-network.rules
@@ -1,5 +1,5 @@
 # old udev has ENV{ID_NET_DRIVER}
-SUBSYSTEMS=="xen", KERNEL=="eth*", ACTION=="add", ENV{ID_NET_DRIVER}=="vif", RUN+="/usr/lib/qubes/setup-ip"
-SUBSYSTEMS=="net", KERNEL=="eth*", ACTION=="remove", ENV{ID_NET_DRIVER}=="vif", RUN+="/usr/lib/qubes/setup-ip"
+SUBSYSTEMS=="xen", KERNEL=="eth*", ACTION=="add", ENV{ID_NET_DRIVER}=="vif", ENV{SYSTEMD_WANTS}+="qubes-network-uplink@%k.service"
+SUBSYSTEMS=="net", KERNEL=="eth*", ACTION=="remove", ENV{ID_NET_DRIVER}=="vif", ENV{SYSTEMD_WANTS}+="qubes-network-uplink@%k.service"
 # new udev has DRIVERS
-SUBSYSTEMS=="xen", KERNEL=="eth*", ACTION=="add", DRIVERS=="vif", RUN+="/usr/lib/qubes/setup-ip"
+SUBSYSTEMS=="xen", KERNEL=="eth*", ACTION=="add", DRIVERS=="vif", ENV{SYSTEMD_WANTS}+="qubes-network-uplink@%k.service"
diff --git a/rpm_spec/core-agent.spec.in b/rpm_spec/core-agent.spec.in
index 3e663d3..17720ab 100644
--- a/rpm_spec/core-agent.spec.in
+++ b/rpm_spec/core-agent.spec.in
@@ -797,9 +797,12 @@ rm -f %{name}-%{version}
 /lib/systemd/system/qubes-firewall.service
 /lib/systemd/system/qubes-iptables.service
 /lib/systemd/system/qubes-network.service
+/lib/systemd/system/qubes-network-uplink.service
+/lib/systemd/system/qubes-network-uplink@.service
 /lib/systemd/system/qubes-updates-proxy.service
 /usr/lib/qubes/init/network-proxy-setup.sh
 /usr/lib/qubes/init/network-proxy-stop.sh
+/usr/lib/qubes/init/network-uplink-wait.sh
 /usr/lib/qubes/init/qubes-iptables
 /usr/lib/qubes/iptables-updates-proxy
 /usr/lib/qubes/qubes-setup-dnat-to-ns
diff --git a/vm-systemd/75-qubes-vm.preset b/vm-systemd/75-qubes-vm.preset
index a2f6cd8..c0fcbe3 100644
--- a/vm-systemd/75-qubes-vm.preset
+++ b/vm-systemd/75-qubes-vm.preset
@@ -91,6 +91,7 @@ enable qubes-update-check.timer
 enable qubes-misc-post.service
 enable qubes-updates-proxy.service
 enable qubes-network.service
+enable qubes-network-uplink.service
 enable qubes-qrexec-agent.service
 enable qubes-mount-dirs.service
 enable qubes-rootfs-resize.service
diff --git a/vm-systemd/misc-post.sh b/vm-systemd/misc-post.sh
index f284efd..1ffaae0 100755
--- a/vm-systemd/misc-post.sh
+++ b/vm-systemd/misc-post.sh
@@ -11,15 +11,6 @@ if [ -n "$(ls -A /usr/local/lib 2>/dev/null)" ] || \
     ldconfig
 fi
 
-# Set IP address again (besides action in udev rules); this is needed by
-# DispVM (to override DispVM-template IP) and in case when qubes-ip was
-# called by udev before loading evtchn kernel module - in which case
-# qubesdb-read fails
-QUBES_MANAGED_IFACE="$(get_qubes_managed_iface)"
-if [ "x$QUBES_MANAGED_IFACE" != "x" ]; then
-INTERFACE="$QUBES_MANAGED_IFACE" ACTION="add" /usr/lib/qubes/setup-ip
-fi
-
 if [ -x /rw/config/rc.local ] ; then
     /rw/config/rc.local
 fi
diff --git a/vm-systemd/network-uplink-wait.sh b/vm-systemd/network-uplink-wait.sh
new file mode 100644
index 0000000..bb0bc34
--- /dev/null
+++ b/vm-systemd/network-uplink-wait.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# Source Qubes library.
+# shellcheck source=init/functions
+. /usr/lib/qubes/init/functions
+
+# Setup IP address at specific time of system boot, instead of asynchronously
+# by udev
+QUBES_MANAGED_IFACE="$(get_qubes_managed_iface)"
+if [ "x$QUBES_MANAGED_IFACE" != "x" ]; then
+    # systemd does not support conditional After= dependencies, nor a tool to
+    # just wait for the unit to be activated
+    # if the network interface is expected, use `systemctl start` to wait for
+    # it to be started - it would be started by udev (SYSTEMD_WANTS) anyway
+    systemctl start "qubes-network-uplink@$QUBES_MANAGED_IFACE.service"
+fi
diff --git a/vm-systemd/qubes-network-uplink.service b/vm-systemd/qubes-network-uplink.service
new file mode 100644
index 0000000..acf8649
--- /dev/null
+++ b/vm-systemd/qubes-network-uplink.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Qubes network uplink wait
+Before=network.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/lib/qubes/init/network-uplink-wait.sh
+
+[Install]
+WantedBy=multi-user.target
diff --git a/vm-systemd/qubes-network-uplink@.service b/vm-systemd/qubes-network-uplink@.service
new file mode 100644
index 0000000..74bf689
--- /dev/null
+++ b/vm-systemd/qubes-network-uplink@.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Qubes network uplink (%i) setup
+After=network-pre.target qubes-iptables.service
+After=sys-subsystem-net-devices-%i.device
+BindsTo=sys-subsystem-net-devices-%i.device
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/lib/qubes/setup-ip add "%i"
+ExecStop=/usr/lib/qubes/setup-ip remove "%i"

From 6aa2b89fba734305aa14d0a50833df414a722383 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 3 Dec 2020 20:51:49 +0100
Subject: [PATCH 20/25] Cleanup setup-ip script a bit

There is no longer a case where $INTERFACE is not set.
---
 network/setup-ip | 121 +++++++++++++++++++++++------------------------
 1 file changed, 59 insertions(+), 62 deletions(-)

diff --git a/network/setup-ip b/network/setup-ip
index fd2db18..07c8568 100755
--- a/network/setup-ip
+++ b/network/setup-ip
@@ -181,75 +181,72 @@ if [ -z "$INTERFACE" ]; then
     exit 1
 fi
 
-if [ -n "$INTERFACE" ]; then
-    if [ "$ACTION" == "add" ]; then
-        MAC="$(get_mac_from_iface "$INTERFACE")"
-        if [ -n "$MAC" ]; then
-            ip="$(/usr/bin/qubesdb-read "/net-config/$MAC/ip" 2> /dev/null)" || ip=
-            ip6="$(/usr/bin/qubesdb-read "/net-config/$MAC/ip6" 2> /dev/null)" || ip6=
-            netmask="$(/usr/bin/qubesdb-read "/net-config/$MAC/netmask" 2> /dev/null)" || netmask=
-            netmask6="$(/usr/bin/qubesdb-read "/net-config/$MAC/netmask6" 2> /dev/null)" || netmask6=
-            gateway="$(/usr/bin/qubesdb-read "/net-config/$MAC/gateway" 2> /dev/null)" || gateway=
-            gateway6="$(/usr/bin/qubesdb-read "/net-config/$MAC/gateway6" 2> /dev/null)" || gateway6=
+if [ "$ACTION" == "add" ]; then
+    MAC="$(get_mac_from_iface "$INTERFACE")"
+    if [ -n "$MAC" ]; then
+        ip="$(/usr/bin/qubesdb-read "/net-config/$MAC/ip" 2> /dev/null)" || ip=
+        ip6="$(/usr/bin/qubesdb-read "/net-config/$MAC/ip6" 2> /dev/null)" || ip6=
+        netmask="$(/usr/bin/qubesdb-read "/net-config/$MAC/netmask" 2> /dev/null)" || netmask=
+        netmask6="$(/usr/bin/qubesdb-read "/net-config/$MAC/netmask6" 2> /dev/null)" || netmask6=
+        gateway="$(/usr/bin/qubesdb-read "/net-config/$MAC/gateway" 2> /dev/null)" || gateway=
+        gateway6="$(/usr/bin/qubesdb-read "/net-config/$MAC/gateway6" 2> /dev/null)" || gateway6=
 
-            # Handle legacy values
-            LEGACY_MAC="$(/usr/bin/qubesdb-read /qubes-mac 2> /dev/null)" || LEGACY_MAC=
-            if [ "$MAC" == "$LEGACY_MAC" ] || [ -z "$LEGACY_MAC" ]; then
-                if [ -z "$ip" ]; then
-                    ip="$(/usr/bin/qubesdb-read /qubes-ip 2> /dev/null)" || ip=
-                fi
-                if [ -z "$ip6" ]; then
-                    ip6="$(/usr/bin/qubesdb-read /qubes-ip6 2> /dev/null)" || ip6=
-                fi
-                if [ -z "$gateway" ]; then
-                    gateway="$(/usr/bin/qubesdb-read /qubes-gateway 2> /dev/null)" || gateway=
-                fi
-                if [ -z "$gateway6" ]; then
-                    gateway6="$(/usr/bin/qubesdb-read /qubes-gateway6 2> /dev/null)" || gateway6=
-                fi
+        # Handle legacy values
+        LEGACY_MAC="$(/usr/bin/qubesdb-read /qubes-mac 2> /dev/null)" || LEGACY_MAC=
+        if [ "$MAC" == "$LEGACY_MAC" ] || [ -z "$LEGACY_MAC" ]; then
+            if [ -z "$ip" ]; then
+                ip="$(/usr/bin/qubesdb-read /qubes-ip 2> /dev/null)" || ip=
             fi
-
-            if [ -z "$netmask" ]; then
-                netmask="255.255.255.255"
+            if [ -z "$ip6" ]; then
+                ip6="$(/usr/bin/qubesdb-read /qubes-ip6 2> /dev/null)" || ip6=
             fi
-            if [ -z "$netmask6" ]; then
-                netmask6="128"
+            if [ -z "$gateway" ]; then
+                gateway="$(/usr/bin/qubesdb-read /qubes-gateway 2> /dev/null)" || gateway=
             fi
-
-            primary_dns=$(/usr/bin/qubesdb-read /qubes-primary-dns 2>/dev/null) || primary_dns=
-            secondary_dns=$(/usr/bin/qubesdb-read /qubes-secondary-dns 2>/dev/null) || secondary_dns=
-
-            if [ -n "$ip" ]; then
-                /sbin/ethtool -K "$INTERFACE" sg off
-                /sbin/ethtool -K "$INTERFACE" tx off
-
-                # If NetworkManager is enabled, let it configure the network
-                if qsvc network-manager && [ -e /usr/bin/nmcli ]; then
-                    configure_network_nm "$MAC" "$INTERFACE" "$ip" "$ip6" "$netmask" "$netmask6" "$gateway" "$gateway6" "$primary_dns" "$secondary_dns"
-                else
-                    configure_network "$MAC" "$INTERFACE" "$ip" "$ip6" "$netmask" "$netmask6" "$gateway" "$gateway6" "$primary_dns" "$secondary_dns"
-                fi
-
-                network=$(qubesdb-read /qubes-netvm-network 2>/dev/null) || network=
-                if [ -n "$network" ]; then
-                    if ! qsvc disable-dns-server; then
-                        configure_qubes_ns
-                    fi
-                    qubes_ip_change_hook
-                fi
+            if [ -z "$gateway6" ]; then
+                gateway6="$(/usr/bin/qubesdb-read /qubes-gateway6 2> /dev/null)" || gateway6=
             fi
         fi
-    elif [ "$ACTION" == "remove" ]; then
-        # make sure network is disabled, especially on shutdown, to prevent
-        # leaks when firewall will get stopped too
-        ip link set "$INTERFACE" down 2>/dev/null || :
 
-        # If exists, we delete NetworkManager configuration file to prevent duplicate entries
-        nm_config="/etc/NetworkManager/system-connections/qubes-uplink-$INTERFACE"
-        rm -rf "$nm_config"
-    else
-        echo "Invalid action '$ACTION'" >&2
-        exit 1
+        if [ -z "$netmask" ]; then
+            netmask="255.255.255.255"
+        fi
+        if [ -z "$netmask6" ]; then
+            netmask6="128"
+        fi
+
+        primary_dns=$(/usr/bin/qubesdb-read /qubes-primary-dns 2>/dev/null) || primary_dns=
+        secondary_dns=$(/usr/bin/qubesdb-read /qubes-secondary-dns 2>/dev/null) || secondary_dns=
+
+        if [ -n "$ip" ]; then
+            /sbin/ethtool -K "$INTERFACE" sg off
+            /sbin/ethtool -K "$INTERFACE" tx off
+
+            # If NetworkManager is enabled, let it configure the network
+            if qsvc network-manager && [ -e /usr/bin/nmcli ]; then
+                configure_network_nm "$MAC" "$INTERFACE" "$ip" "$ip6" "$netmask" "$netmask6" "$gateway" "$gateway6" "$primary_dns" "$secondary_dns"
+            else
+                configure_network "$MAC" "$INTERFACE" "$ip" "$ip6" "$netmask" "$netmask6" "$gateway" "$gateway6" "$primary_dns" "$secondary_dns"
+            fi
+
+            network=$(qubesdb-read /qubes-netvm-network 2>/dev/null) || network=
+            if [ -n "$network" ]; then
+                if ! qsvc disable-dns-server; then
+                    configure_qubes_ns
+                fi
+                qubes_ip_change_hook
+            fi
+        fi
     fi
+elif [ "$ACTION" == "remove" ]; then
+    # make sure network is disabled, especially on shutdown, to prevent
+    # leaks when firewall will get stopped too
+    ip link set "$INTERFACE" down 2>/dev/null || :
 
+    # If exists, we delete NetworkManager configuration file to prevent duplicate entries
+    nm_config="/etc/NetworkManager/system-connections/qubes-uplink-$INTERFACE"
+    rm -rf "$nm_config"
+else
+    echo "Invalid action '$ACTION'" >&2
+    exit 1
 fi

From 8a3cd3db1d3ec28d505ffbcb6d48ee2b6536a4a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 4 Dec 2020 03:23:18 +0100
Subject: [PATCH 21/25] Make init/functions suitable for running with 'set -u'

Initialize local variables.
---
 init/functions | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/init/functions b/init/functions
index 2582b4d..b9c44c4 100644
--- a/init/functions
+++ b/init/functions
@@ -118,7 +118,7 @@ umount_retry() {
 
 get_mac_from_iface() {
     local iface="$1"
-    local mac
+    local mac=
     if [ "x$iface" != "x" ] && [ -e "/sys/class/net/$iface" ]; then
         mac="$(cat "/sys/class/net/$iface/address")"
     fi
@@ -127,7 +127,7 @@ get_mac_from_iface() {
 
 get_iface_from_mac() {
     local mac="$1"
-    local iface
+    local iface=
     if [ "x$mac" != "x" ]; then
         iface="$(ip -o link | grep -i "$mac" | awk '{print $2}' | cut -d ':' -f1)"
     fi

From 519e82b7c03f3a12cf05b142e38ca84ca70b7f09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 4 Dec 2020 12:28:27 +0100
Subject: [PATCH 22/25] init/functions: do not guess 'eth0' as Qubes-managed
 interface

... if it doesn't exist.
The /qubes-mac qubesdb entry is present on Qubes 4.1, but not 4.0. It is
ok to depend on it here, but keep safer fallback if this code would need
to be backported.
---
 init/functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/init/functions b/init/functions
index b9c44c4..ba05485 100644
--- a/init/functions
+++ b/init/functions
@@ -141,7 +141,7 @@ get_qubes_managed_iface() {
     qubes_iface="$(get_iface_from_mac "$mac")"
     if [ "x$qubes_iface" != "x" ]; then
         echo "$qubes_iface"
-    else
+    elif [ -e /sys/class/net/eth0 ]; then
         echo eth0
     fi
 }

From 66b3e628f2bf0ec8f23b0b42484d014e5cad23bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 5 Dec 2020 18:13:27 +0100
Subject: [PATCH 23/25] Order NetworkManager after qubes-network-uplink.service

Make sure NM config for uplink interface (eth0) is created before
starting NetworkManager itself. Otherwise NM helpfully will try to use
automatic DHCP configuration, which will fail and cause delays on
network start.
---
 vm-systemd/NetworkManager.service.d/30_qubes.conf | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/vm-systemd/NetworkManager.service.d/30_qubes.conf b/vm-systemd/NetworkManager.service.d/30_qubes.conf
index 047cc22..5f598a1 100644
--- a/vm-systemd/NetworkManager.service.d/30_qubes.conf
+++ b/vm-systemd/NetworkManager.service.d/30_qubes.conf
@@ -4,6 +4,8 @@ ConditionPathExists=/var/run/qubes-service/network-manager
 After=qubes-mount-dirs.service
 # For /var/run/qubes-service
 After=qubes-sysinit.service
+# For configuration of qubes-provided interfaces
+After=qubes-network-uplink.service
 
 [Service]
 ExecStartPre=/usr/lib/qubes/network-manager-prepare-conf-dir

From edde0d573e9dbfad50189db689267997c95eb3a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Pierret=20=28fepitre=29?=
 <frederic.pierret@qubes-os.org>
Date: Sat, 12 Dec 2020 11:11:18 +0100
Subject: [PATCH 24/25] debian: update control

---
 debian/control | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/debian/control b/debian/control
index d02dd12..6984956 100644
--- a/debian/control
+++ b/debian/control
@@ -3,22 +3,21 @@ Section: admin
 Priority: extra
 Maintainer: unman <unman@thirdeyesecurity.org>
 Build-Depends:
+    debhelper,
     libpam0g-dev,
     libqubes-rpc-filecopy-dev (>= 3.1.3),
     libvchan-xen-dev,
     python3,
     python3-setuptools,
-    debhelper,
     quilt,
     libxen-dev,
     pkg-config,
-    dh-systemd (>= 1.5),
     dh-python,
     lsb-release,
     xserver-xorg-dev,
     config-package-dev,
     pandoc,
-Standards-Version: 3.9.5
+Standards-Version: 4.4.0.1
 Homepage: https://www.qubes-os.org
 Vcs-Git: https://github.com/QubesOS/qubes-core-agent-linux
 

From b15ff53bc6dee36cecf28413554fb7c856ae0517 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Pierret=20=28fepitre=29?=
 <frederic.pierret@qubes-os.org>
Date: Sat, 12 Dec 2020 11:44:47 +0100
Subject: [PATCH 25/25] debian: update compat

---
 debian/compat | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/compat b/debian/compat
index ec63514..f599e28 100644
--- a/debian/compat
+++ b/debian/compat
@@ -1 +1 @@
-9
+10