Memory management across VMs, first release

appvm/qubes_core

@@ -87,6 +87,7 @@ start()
         fi
 	fi
 
+	/usr/lib/qubes/meminfo-writer &
 	[ -x /rw/config/rc.local ] && /rw/config/rc.local
 	success
 	echo ""

common/meminfo-writer

@@ -0,0 +1,4 @@
+#!/bin/sh
+while sleep 1 ; do
+	xenstore-write memory/meminfo "`cat /proc/meminfo`"
+done

dom0/init.d/qubes_core

@@ -56,6 +56,8 @@ start()
     xm mem-set 0 1600
     cp /var/lib/qubes/qubes.xml /var/lib/qubes/backup/qubes-$(date +%F-%T).xml
     setup_dvm_files
+    /usr/lib/qubes/qmemman_daemon.py >/var/log/qubes/qmemman.log 2>/var/log/qubes/qmemman.errs & 
+    /usr/lib/qubes/meminfo-writer &
     touch /var/lock/subsys/qubes_core
     success
     echo

dom0/pendrive_swapper/qfilexchgd

@@ -30,6 +30,7 @@ import time
 from qubes.qubes import QubesVmCollection
 from qubes.qubes import QubesException
 from qubes.qubes import QubesDaemonPidfile
+from qubes.qmemman_client import QMemmanClient
 
 filename_seq = 50
 pen_cmd = '/usr/lib/qubes/qubes_pencmd'
@@ -187,13 +188,11 @@ class DomainState:
 
     def handle_transfer_disposable(self, transaction_seq):
 
-        mem_for_dvm = 400
+        qmemman_client = QMemmanClient()
-        xenfreepages_s = subprocess.Popen(["/usr/lib/qubes/xenfreepages"],stdout=subprocess.PIPE).stdout.readline()
+        if not qmemman_client.request_memory(400*1024*1024):
-        xenfree_mb = int(xenfreepages_s)*4096/1024/1024
+            qmemman_client.close()
-        if xenfree_mb < mem_for_dvm:
+            errmsg = 'Not enough memory to create DVM. '
-            errmsg = 'Not enough memory to create DVM: '
+            errmsg +='Terminate some appVM and retry.'
-            errmsg +='have ' + str(xenfree_mb) + 'MB, need '
-            errmsg +=str(mem_for_dvm) + 'MB. Terminate some appVM and retry.'
             subprocess.call(['/usr/bin/kdialog', '--sorry', errmsg])
             return False
 
@@ -205,12 +204,14 @@ class DomainState:
         if vm is None:
             logproc( 'Domain ' + vmname + ' does not exist ?')
             qvm_collection.unlock_db()
+            qmemman_client.close()
             return False
         retcode = subprocess.call(['/usr/lib/qubes/qubes_restore',
             current_savefile,
             '-c', vm.label.color,
             '-i', vm.label.icon,
             '-l', str(vm.label.index)])
+        qmemman_client.close()
         if retcode != 0:
             subprocess.call(['/usr/bin/kdialog', '--sorry', 'DisposableVM creation failed, see qubes_restore.log'])
             qvm_collection.unlock_db()

dom0/qmemman/qmemman.py

@@ -0,0 +1,185 @@
+import xmlrpclib
+from xen.xm import XenAPI
+import xen.lowlevel.xc
+import string
+import time
+import qmemman_algo
+import os
+
+class XendSession(object):
+    def __init__(self):
+#        self.get_xend_session_old_api()
+        self.get_xend_session_new_api()
+
+#    def get_xend_session_old_api(self):
+#        from xen.xend import XendClient
+#        from xen.util.xmlrpcclient import ServerProxy
+#        self.xend_server = ServerProxy(XendClient.uri)
+#        if self.xend_server is None:
+#            print "get_xend_session_old_api(): cannot open session!"
+
+
+    def get_xend_session_new_api(self):
+        xend_socket_uri = "httpu:///var/run/xend/xen-api.sock"
+        self.session = XenAPI.Session (xend_socket_uri)
+        self.session.login_with_password ("", "")
+        if self.session is None:
+            print "get_xend_session_new_api(): cannot open session!"
+
+class DomainState:
+    def __init__(self, id):
+        self.meminfo = None
+        self.memory_actual = None
+        self.mem_used = None
+        self.uuid = None
+        self.id = id
+        self.meminfo_updated = False
+
+class SystemState:
+    def __init__(self):
+        self.xend_session = XendSession()
+        self.domdict = {}
+        self.xc = xen.lowlevel.xc.xc()
+        self.BALOON_DELAY = 0.1
+
+    def add_domain(self, id):
+        self.domdict[id] = DomainState(id)
+
+    def del_domain(self, id):
+        self.domdict.pop(id)
+
+    def get_free_xen_memory(self):
+        return self.xc.physinfo()['free_memory']*1024
+#        hosts = self.xend_session.session.xenapi.host.get_all()
+#        host_record = self.xend_session.session.xenapi.host.get_record(hosts[0])
+#        host_metrics_record = self.xend_session.session.xenapi.host_metrics.get_record(host_record["metrics"])
+#        ret = host_metrics_record["memory_free"]
+#        return long(ret)
+
+    def refresh_memactual(self):
+        update_uuid_info = False
+        for domain in self.xc.domain_getinfo():
+            id = str(domain['domid'])
+            if self.domdict.has_key(id):
+                self.domdict[id].memory_actual = domain['mem_kb']*1024
+                if self.domdict[id].uuid is None:
+                    update_uuid_info = True
+        if not update_uuid_info:
+            return
+        dom_recs = self.xend_session.session.xenapi.VM.get_all_records()
+#        dom_metrics_recs = self.xend_session.session.xenapi.VM_metrics.get_all_records()
+        for dom_ref, dom_rec in dom_recs.items():
+#            dom_metrics_rec = dom_metrics_recs[dom_rec['metrics']]
+            id = dom_rec['domid']
+#            mem = int(dom_metrics_rec['memory_actual'])/1024
+            if (self.domdict.has_key(id)):
+#                self.domdict[id].memory_actual = mem
+                self.domdict[id].uuid = dom_rec['uuid']   
+
+    def parse_meminfo(self, meminfo):
+        dict = {}
+        l1 = string.split(meminfo,"\n")
+        for i in l1:
+            l2 = string.split(i)
+            if len(l2) >= 2:
+                dict[string.rstrip(l2[0], ":")] = l2[1]
+
+        try:
+            for i in ('MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree'):
+                val = int(dict[i])*1024
+                if (val < 0):
+                    return None
+                dict[i] = val
+        except:
+            return None
+
+        if dict['SwapTotal'] < dict['SwapFree']:
+            return None
+        return dict
+
+#the below works (and is fast), but then 'xm list' shows unchanged memory value
+    def mem_set_alternative(self, id, val):
+        os.system('xenstore-write /local/domain/' + id + '/memory/target ' + str(val/1024))
+        self.xc.domain_set_target_mem(int(id), val/1024)
+    
+    def mem_set(self, id, val):
+        uuid = self.domdict[id].uuid
+        print 'mem-set domain', id, 'to', val
+        self.xend_session.session.xenapi.VM.set_memory_dynamic_max_live(uuid, val)
+        self.xend_session.session.xenapi.VM.set_memory_dynamic_min_live(uuid, val)
+
+    def do_balloon(self, memsize):
+        MAX_TRIES = 20
+        niter = 0
+        prev_memory_actual = None
+        for i in self.domdict.keys():
+            self.domdict[i].no_progress = False
+        while True:
+            xenfree = self.get_free_xen_memory()
+            print 'got xenfree=', xenfree
+            if xenfree >= memsize:
+                return True
+            self.refresh_memactual()
+            if prev_memory_actual is not None:
+                for i in prev_memory_actual.keys():
+                    if prev_memory_actual[i] == self.domdict[i].memory_actual:
+                        self.domdict[i].no_progress = True
+                        print 'domain', i, 'stuck at', self.domdict[i].memory_actual
+            memset_reqs = qmemman_algo.balloon(memsize-xenfree, self.domdict)
+            print 'requests:', memset_reqs
+            if niter > MAX_TRIES or len(memset_reqs) == 0:
+                return False
+            prev_memory_actual = {}
+            for i in memset_reqs:
+                dom, mem = i
+                self.mem_set(dom, mem)
+                prev_memory_actual[dom] = self.domdict[dom].memory_actual
+            time.sleep(self.BALOON_DELAY)
+            niter = niter + 1
+            
+    def refresh_meminfo(self, domid, val):
+        self.domdict[domid].meminfo = self.parse_meminfo(val)
+        self.domdict[domid].meminfo_updated = True
+
+    def adjust_inflates_to_xenfree(self, reqs, idx):
+        i = idx
+        memory_needed = 0
+        while i < len(reqs):
+            dom, mem = reqs[i]
+            memory_needed += mem - self.domdict[dom].memory_actual
+            i = i + 1
+        scale = 1.0*self.get_free_xen_memory()/memory_needed
+        dom, mem = reqs[idx]
+        scaled_req = self.domdict[dom].memory_actual + scale*(mem - self.domdict[dom].memory_actual)
+        return int(scaled_req)
+
+    def do_balance(self):
+        if os.path.isfile('/etc/do-not-membalance'):
+            return
+        self.refresh_memactual()
+        xenfree = self.get_free_xen_memory()
+        memset_reqs = qmemman_algo.balance(xenfree, self.domdict)
+        wait_before_first_inflate = False
+        i = 0
+        while i < len(memset_reqs):
+            dom, mem = memset_reqs[i]
+            memory_change = mem - self.domdict[dom].memory_actual
+            if abs(memory_change) < 100*1024*1024:
+                i = i + 1
+                continue
+            if memory_change < 0:
+                wait_before_first_inflate = True
+            else:
+                if wait_before_first_inflate:
+                    time.sleep(self.BALOON_DELAY)
+                    wait_before_first_inflate = False
+                #the following is called before _each_ inflate, to account for possibility that
+                #previously triggered memory release is in progress
+                mem = self.adjust_inflates_to_xenfree(memset_reqs, i)
+            self.mem_set(dom, mem)
+            i = i + 1
+
+#        for i in self.domdict.keys():
+#            print 'domain ', i, ' meminfo=', self.domdict[i].meminfo, 'actual mem', self.domdict[i].memory_actual
+#            print 'domain ', i, 'actual mem', self.domdict[i].memory_actual
+#        print 'xen free mem', self.get_free_xen_memory()

dom0/qmemman/qmemman_algo.py

@@ -0,0 +1,101 @@
+def is_suspicious(dom):
+    ret = False
+    if dom.meminfo['SwapTotal'] < dom.meminfo['SwapFree']:
+        ret = True
+    if dom.memory_actual < dom.meminfo['MemFree'] + dom.meminfo['Cached'] + dom.meminfo['Buffers']:
+        ret = True
+    if ret:
+        print 'suspicious meminfo for domain', dom.id, 'mem actual', dom.memory_actual, dom.meminfo
+    return ret
+
+def recalc_mem_used(domdict):
+    for domid in domdict.keys():
+        dom = domdict[domid]
+        if dom.meminfo_updated:
+            dom.meminfo_updated = False
+            if is_suspicious(dom):
+                dom.meminfo = None
+                dom.mem_used = None
+            else:
+                dom.mem_used =  dom.memory_actual - dom.meminfo['MemFree'] - dom.meminfo['Cached'] - dom.meminfo['Buffers'] + dom.meminfo['SwapTotal'] - dom.meminfo['SwapFree']
+
+def prefmem(dom):
+    if dom.meminfo_updated:
+        raise AssertionError('meminfo_updated=True in prefmem')
+    CACHE_FACTOR = 1.3
+#dom0 is special, as it must have large cache, for vbds. Thus, give it a special boost
+    if dom.id == '0':
+        return dom.mem_used*CACHE_FACTOR + 350*1024*1024
+    return dom.mem_used*CACHE_FACTOR
+
+def memneeded(dom):
+#do not change
+#in balance(), "distribute totalsum proportionally to mempref" relies on this exact formula
+    ret = prefmem(dom) - dom.memory_actual
+    return ret
+    
+    
+def balloon(memsize, domdict):
+    REQ_SAFETY_NET_FACTOR = 1.05
+    donors = list()
+    request = list()
+    available = 0
+    recalc_mem_used(domdict)
+    for i in domdict.keys():
+        if domdict[i].meminfo is None:
+            continue
+        if domdict[i].no_progress:
+            continue
+        need = memneeded(domdict[i])
+        if need < 0:
+            print 'balloon: dom' , i, 'has actual memory', domdict[i].memory_actual
+            donors.append((i,-need))
+            available-=need   
+    print 'req=', memsize, 'avail=', available, 'donors', donors
+    if available<memsize:
+        return ()
+    scale = 1.0*memsize/available
+    for donors_iter in donors:
+        id, mem = donors_iter
+        memborrowed = mem*scale*REQ_SAFETY_NET_FACTOR
+        print 'borrow' , memborrowed, 'from', id
+        memtarget = int(domdict[id].memory_actual - memborrowed)
+        request.append((id, memtarget))
+    return request
+# REQ_SAFETY_NET_FACTOR is a bit greater that 1. So that if the domain yields a bit less than requested, due
+# to e.g. rounding errors, we will not get stuck. The surplus will return to the VM during "balance" call.
+ 
+def balance(xenfree, domdict):
+    total_memneeded = 0
+    total_mem_pref = 0
+    recalc_mem_used(domdict)
+#pass 1: compute the above "total" values
+    for i in domdict.keys():
+        if domdict[i].meminfo is None:
+            continue
+        need = memneeded(domdict[i])
+        print 'domain' , i, 'act/pref', domdict[i].memory_actual, prefmem(domdict[i]), 'need=', need
+        total_memneeded += need
+        total_mem_pref += prefmem(domdict[i])
+
+    totalsum = xenfree - total_memneeded  
+
+#pass 2: redistribute "totalsum" of memory between domains, proportionally to prefmem
+    donors = list()
+    acceptors = list()
+    for i in domdict.keys():
+        if domdict[i].meminfo is None:
+            continue
+#distribute totalsum proportionally to mempref
+        scale = 1.0*prefmem(domdict[i])/total_mem_pref
+        target_nonint = prefmem(domdict[i]) + scale*totalsum
+#prevent rounding errors
+        target = int(0.995*target_nonint)
+        if (target < domdict[i].memory_actual):
+            donors.append((i, target))
+        else:
+            acceptors.append((i, target))
+    print 'balance: xenfree=', xenfree, 'requests:', donors + acceptors
+    return donors + acceptors
+    
+    

dom0/qmemman/qmemman_client.py

@@ -0,0 +1,16 @@
+import socket
+
+class QMemmanClient:
+
+    def request_memory(self, amount):
+        self.sock = socket.socket(socket.AF_UNIX)
+        self.sock.connect("/var/run/qubes/qmemman.sock")
+        self.sock.send(str(amount)+"\n")
+        self.received = self.sock.recv(1024).strip()
+        if self.received == 'OK':
+            return True
+        else:
+            return False
+
+    def close(self):
+        self.sock.close()

dom0/qmemman/qmemman_server.py

@@ -0,0 +1,123 @@
+#!/usr/bin/python
+import SocketServer
+import thread
+import time
+import xen.lowlevel.xs
+import sys
+import os
+from qmemman import SystemState
+
+system_state = SystemState()
+global_lock = thread.allocate_lock()
+additional_balance_delay = 0
+
+def only_in_first_list(l1, l2):
+    ret=[]
+    for i in l1:
+        if not i in l2:
+            ret.append(i)
+    return ret
+
+def get_req_node(domain_id):
+    return '/local/domain/'+domain_id+'/memory/meminfo'
+
+                    
+class WatchType:
+    def __init__(self, fn, param):
+        self.fn = fn
+        self.param = param
+
+class XS_Watcher:
+    def __init__(self):
+        self.handle = xen.lowlevel.xs.xs()
+        self.handle.watch('/local/domain', WatchType(XS_Watcher.dom_list_change, None))
+        self.watch_token_dict = {}
+
+    def dom_list_change(self, param):
+        curr = self.handle.ls('', '/local/domain')
+        if curr == None:
+            return
+        global_lock.acquire()
+        for i in only_in_first_list(curr, self.watch_token_dict.keys()):
+            watch = WatchType(XS_Watcher.request, i)
+            self.watch_token_dict[i] = watch
+            self.handle.watch(get_req_node(i), watch)
+            system_state.add_domain(i)
+        for i in only_in_first_list(self.watch_token_dict.keys(), curr):
+            self.handle.unwatch(get_req_node(i), self.watch_token_dict[i])
+            self.watch_token_dict.pop(i)
+            system_state.del_domain(i)
+        global_lock.release()
+
+    def request(self, domain_id):
+        ret = self.handle.read('', get_req_node(domain_id))
+        if ret == None or ret == '':
+            return
+        global_lock.acquire()
+        system_state.refresh_meminfo(domain_id, ret)
+        global_lock.release()
+
+    def watch_loop(self):
+#        sys.stderr = file('/var/log/qubes/qfileexchgd.errors', 'a')
+        while True:
+            result = self.handle.read_watch()
+            token = result[1]
+            token.fn(self, token.param)
+
+
+class QMemmanReqHandler(SocketServer.BaseRequestHandler):
+    """
+    The RequestHandler class for our server.
+
+    It is instantiated once per connection to the server, and must
+    override the handle() method to implement communication to the
+    client.
+    """
+
+    def handle(self):
+        # self.request is the TCP socket connected to the client
+        while True:
+            self.data = self.request.recv(1024).strip()
+            if len(self.data) == 0:
+                print 'EOF'
+                return
+            if self.data == "DONE":
+                return
+            global_lock.acquire()
+            if system_state.do_balloon(int(self.data)):
+                resp = "OK\n"
+                additional_balance_delay = 5
+            else:
+                resp = "FAIL\n"
+            global_lock.release()
+            self.request.send(resp)
+
+
+def start_server():
+    SOCK_PATH='/var/run/qubes/qmemman.sock'
+    try:
+        os.unlink(SOCK_PATH)
+    except:
+        pass
+    os.umask(0)
+    server = SocketServer.UnixStreamServer(SOCK_PATH, QMemmanReqHandler)
+    os.umask(077)
+    server.serve_forever()
+
+def start_balancer():
+    while True:
+        time.sleep(1)
+        if additional_balance_delay == 0:
+            time.sleep(additional_balance_delay)
+            additional_balance_delay = 0
+        global_lock.acquire()
+        if additional_balance_delay == 0:
+            system_state.do_balance()
+        global_lock.release()
+
+class QMemmanServer:
+    @staticmethod          
+    def main():
+        thread.start_new_thread(start_server, tuple([]))
+        thread.start_new_thread(start_balancer, tuple([]))
+        XS_Watcher().watch_loop()

dom0/qmemman/server.py

@@ -0,0 +1,4 @@
+#!/usr/bin/python
+from qubes.qmemman_server import QMemmanServer
+
+QMemmanServer.main()

dom0/qvm-core/qubes.py

@@ -29,6 +29,7 @@ import xml.parsers.expat
 import fcntl
 import re
 import shutil
+from qmemman_client import QMemmanClient
 
 # Do not use XenAPI or create/read any VM files
 # This is for testing only!
@@ -326,6 +327,18 @@ class QubesVm(object):
 
         return mem
 
+    def get_mem_dynamic_max(self):
+        if dry_run:
+            return 666
+
+        try:
+            mem = int(xend_session.session.xenapi.VM.get_memory_dynamic_max(self.session_uuid))
+        except XenAPI.Failure:
+            self.refresh_xend_session()
+            mem = int(xend_session.session.xenapi.VM.get_memory_dynamic_max(self.session_uuid))
+
+        return mem
+
 
     def get_cpu_total_load(self):
         if dry_run:
@@ -474,15 +487,11 @@ class QubesVm(object):
         if verbose:
             print "--> Loading the VM (type = {0})...".format(self.type)
 
-        mem_required = self.get_mem_static_max()
+        mem_required = self.get_mem_dynamic_max()
-        dom0_mem = dom0_vm.get_mem()
+        qmemman_client = QMemmanClient()
-        dom0_mem_new = dom0_mem - mem_required + self.get_free_xen_memory()
+        if not qmemman_client.request_memory(mem_required):
-        if verbose:
+            qmemman_client.close()
-            print "--> AppVM required mem     : {0}".format(mem_required)
+            raise MemoryError ("ERROR: insufficient memory to start this VM")
-            print "--> Dom0 mem after launch  : {0}".format(dom0_mem_new)
-
-        if dom0_mem_new < dom0_min_memory:
-            raise MemoryError ("ERROR: starting this VM would cause Dom0 memory to go below {0}B".format(dom0_min_memory))
 
         try:
             xend_session.session.xenapi.VM.start (self.session_uuid, True) # Starting a VM paused
@@ -490,6 +499,8 @@ class QubesVm(object):
             self.refresh_xend_session()
             xend_session.session.xenapi.VM.start (self.session_uuid, True) # Starting a VM paused
 
+        qmemman_client.close() # let qmemman_daemon resume balancing
+        
         xid = int (xend_session.session.xenapi.VM.get_domid (self.session_uuid))
 
         if verbose:

rpm_spec/core-appvm.spec

@@ -65,6 +65,7 @@ cp qubes_timestamp qvm-copy-to-vm qvm-open-in-dvm $RPM_BUILD_ROOT/usr/bin
 mkdir -p $RPM_BUILD_ROOT/usr/lib/qubes
 cp qubes_add_pendrive_script qubes_penctl qvm-copy-to-vm.kde $RPM_BUILD_ROOT/usr/lib/qubes
 ln -s /usr/bin/qvm-open-in-dvm $RPM_BUILD_ROOT/usr/lib/qubes/qvm-dvm-transfer 
+cp ../common/meminfo-writer $RPM_BUILD_ROOT/usr/lib/qubes
 mkdir -p $RPM_BUILD_ROOT/%{kde_service_dir}
 cp qvm-copy.desktop qvm-dvm.desktop $RPM_BUILD_ROOT/%{kde_service_dir}
 mkdir -p $RPM_BUILD_ROOT/etc/udev/rules.d
@@ -187,6 +188,7 @@ rm -rf $RPM_BUILD_ROOT
 /usr/lib/qubes/qvm-copy-to-vm.kde
 %attr(4755,root,root) /usr/bin/qvm-open-in-dvm
 /usr/lib/qubes/qvm-dvm-transfer
+/usr/lib/qubes/meminfo-writer
 %{kde_service_dir}/qvm-copy.desktop
 %{kde_service_dir}/qvm-dvm.desktop
 %attr(4755,root,root) /usr/lib/qubes/qubes_penctl

rpm_spec/core-dom0.spec

@@ -44,8 +44,8 @@ Requires:	python, xen-runtime, pciutils, python-inotify, python-daemon, kernel-q
 The Qubes core files for installation on Dom0.
 
 %build
-python -m compileall qvm-core
+python -m compileall qvm-core qmemman
-python -O -m compileall qvm-core
+python -O -m compileall qvm-core qmemman
 make -C restore
 
 %install
@@ -67,6 +67,8 @@ cp qvm-core/qubes.py $RPM_BUILD_ROOT%{python_sitearch}/qubes
 cp qvm-core/qubes.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes
 cp qvm-core/__init__.py $RPM_BUILD_ROOT%{python_sitearch}/qubes
 cp qvm-core/__init__.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes
+cp qmemman/qmemman*py $RPM_BUILD_ROOT%{python_sitearch}/qubes
+cp qmemman/qmemman*py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes
 
 mkdir -p $RPM_BUILD_ROOT/usr/lib/qubes
 cp aux-tools/patch_appvm_initramfs.sh $RPM_BUILD_ROOT/usr/lib/qubes
@@ -77,6 +79,8 @@ cp aux-tools/convert_dirtemplate2vm.sh $RPM_BUILD_ROOT/usr/lib/qubes
 cp aux-tools/create_apps_for_appvm.sh $RPM_BUILD_ROOT/usr/lib/qubes
 cp aux-tools/remove_appvm_appmenus.sh $RPM_BUILD_ROOT/usr/lib/qubes
 cp pendrive_swapper/qubes_pencmd $RPM_BUILD_ROOT/usr/lib/qubes
+cp qmemman/server.py $RPM_BUILD_ROOT/usr/lib/qubes/qmemman_daemon.py
+cp ../common/meminfo-writer $RPM_BUILD_ROOT/usr/lib/qubes/
 
 cp restore/xenstore-watch restore/qvm-create-default-dvm $RPM_BUILD_ROOT/usr/bin
 cp restore/qubes_restore restore/xenfreepages $RPM_BUILD_ROOT/usr/lib/qubes
@@ -195,6 +199,7 @@ fi
 %{python_sitearch}/qubes/__init__.py
 %{python_sitearch}/qubes/__init__.pyc
 %{python_sitearch}/qubes/__init__.pyo
+%{python_sitearch}/qubes/qmemman*.py*
 /usr/lib/qubes/patch_appvm_initramfs.sh
 /usr/lib/qubes/unbind_pci_device.sh
 /usr/lib/qubes/unbind_all_network_devices
@@ -203,6 +208,8 @@ fi
 /usr/lib/qubes/create_apps_for_appvm.sh
 /usr/lib/qubes/remove_appvm_appmenus.sh
 /usr/lib/qubes/qubes_pencmd
+/usr/lib/qubes/qmemman_daemon.py*
+/usr/lib/qubes/meminfo-writer
 %attr(770,root,qubes) %dir /var/lib/qubes
 %attr(770,root,qubes) %dir /var/lib/qubes/vm-templates
 %attr(770,root,qubes) %dir /var/lib/qubes/appvms