Qubes/core-admin
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Memory management across VMs, first release

Browse Source
This commit is contained in:
Rafal Wojtczuk 2010-08-30 11:40:19 +02:00
parent 8317c2ca18
commit 62487c0f1e
12 changed files with 475 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
appvm/qubes_core
View File

@ -87,6 +87,7 @@ start()
fi
fi
/usr/lib/qubes/meminfo-writer &
[ -x /rw/config/rc.local ] && /rw/config/rc.local
success
echo ""

4
common/meminfo-writer Executable file
View File

@ -0,0 +1,4 @@
#!/bin/sh
while sleep 1 ; do
xenstore-write memory/meminfo "`cat /proc/meminfo`"
done

2
dom0/init.d/qubes_core
View File

@ -56,6 +56,8 @@ start()
xm mem-set 0 1600
cp /var/lib/qubes/qubes.xml /var/lib/qubes/backup/qubes-$(date +%F-%T).xml
setup_dvm_files
/usr/lib/qubes/qmemman_daemon.py >/var/log/qubes/qmemman.log 2>/var/log/qubes/qmemman.errs &
/usr/lib/qubes/meminfo-writer &
touch /var/lock/subsys/qubes_core
success
echo

15
dom0/pendrive_swapper/qfilexchgd
View File

@ -30,6 +30,7 @@ import time
from qubes.qubes import QubesVmCollection
from qubes.qubes import QubesException
from qubes.qubes import QubesDaemonPidfile
from qubes.qmemman_client import QMemmanClient
filename_seq = 50
pen_cmd = '/usr/lib/qubes/qubes_pencmd'
@ -187,13 +188,11 @@ class DomainState:
def handle_transfer_disposable(self, transaction_seq):
mem_for_dvm = 400
xenfreepages_s = subprocess.Popen(["/usr/lib/qubes/xenfreepages"],stdout=subprocess.PIPE).stdout.readline()
xenfree_mb = int(xenfreepages_s)*4096/1024/1024
if xenfree_mb < mem_for_dvm:
errmsg = 'Not enough memory to create DVM: '
errmsg +='have ' + str(xenfree_mb) + 'MB, need '
errmsg +=str(mem_for_dvm) + 'MB. Terminate some appVM and retry.'
qmemman_client = QMemmanClient()
if not qmemman_client.request_memory(400*1024*1024):
qmemman_client.close()
errmsg = 'Not enough memory to create DVM. '
errmsg +='Terminate some appVM and retry.'
subprocess.call(['/usr/bin/kdialog', '--sorry', errmsg])
return False
@ -205,12 +204,14 @@ class DomainState:
if vm is None:
logproc( 'Domain ' + vmname + ' does not exist ?')
qvm_collection.unlock_db()
qmemman_client.close()
return False
retcode = subprocess.call(['/usr/lib/qubes/qubes_restore',
current_savefile,
'-c', vm.label.color,
'-i', vm.label.icon,
'-l', str(vm.label.index)])
qmemman_client.close()
if retcode != 0:
subprocess.call(['/usr/bin/kdialog', '--sorry', 'DisposableVM creation failed, see qubes_restore.log'])
qvm_collection.unlock_db()

185
dom0/qmemman/qmemman.py Executable file
View File

@ -0,0 +1,185 @@
import xmlrpclib
from xen.xm import XenAPI
import xen.lowlevel.xc
import string
import time
import qmemman_algo
import os
class XendSession(object):
def __init__(self):
# self.get_xend_session_old_api()
self.get_xend_session_new_api()
# def get_xend_session_old_api(self):
# from xen.xend import XendClient
# from xen.util.xmlrpcclient import ServerProxy
# self.xend_server = ServerProxy(XendClient.uri)
# if self.xend_server is None:
# print "get_xend_session_old_api(): cannot open session!"
def get_xend_session_new_api(self):
xend_socket_uri = "httpu:///var/run/xend/xen-api.sock"
self.session = XenAPI.Session (xend_socket_uri)
self.session.login_with_password ("", "")
if self.session is None:
print "get_xend_session_new_api(): cannot open session!"
class DomainState:
def __init__(self, id):
self.meminfo = None
self.memory_actual = None
self.mem_used = None
self.uuid = None
self.id = id
self.meminfo_updated = False
class SystemState:
def __init__(self):
self.xend_session = XendSession()
self.domdict = {}
self.xc = xen.lowlevel.xc.xc()
self.BALOON_DELAY = 0.1
def add_domain(self, id):
self.domdict[id] = DomainState(id)
def del_domain(self, id):
self.domdict.pop(id)
def get_free_xen_memory(self):
return self.xc.physinfo()['free_memory']*1024
# hosts = self.xend_session.session.xenapi.host.get_all()
# host_record = self.xend_session.session.xenapi.host.get_record(hosts[0])
# host_metrics_record = self.xend_session.session.xenapi.host_metrics.get_record(host_record["metrics"])
# ret = host_metrics_record["memory_free"]
# return long(ret)
def refresh_memactual(self):
update_uuid_info = False
for domain in self.xc.domain_getinfo():
id = str(domain['domid'])
if self.domdict.has_key(id):
self.domdict[id].memory_actual = domain['mem_kb']*1024
if self.domdict[id].uuid is None:
update_uuid_info = True
if not update_uuid_info:
return
dom_recs = self.xend_session.session.xenapi.VM.get_all_records()
# dom_metrics_recs = self.xend_session.session.xenapi.VM_metrics.get_all_records()
for dom_ref, dom_rec in dom_recs.items():
# dom_metrics_rec = dom_metrics_recs[dom_rec['metrics']]
id = dom_rec['domid']
# mem = int(dom_metrics_rec['memory_actual'])/1024
if (self.domdict.has_key(id)):
# self.domdict[id].memory_actual = mem
self.domdict[id].uuid = dom_rec['uuid']
def parse_meminfo(self, meminfo):
dict = {}
l1 = string.split(meminfo,"\n")
for i in l1:
l2 = string.split(i)
if len(l2) >= 2:
dict[string.rstrip(l2[0], ":")] = l2[1]
try:
for i in ('MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree'):
val = int(dict[i])*1024
if (val < 0):
return None
dict[i] = val
except:
return None
if dict['SwapTotal'] < dict['SwapFree']:
return None
return dict
#the below works (and is fast), but then 'xm list' shows unchanged memory value
def mem_set_alternative(self, id, val):
os.system('xenstore-write /local/domain/' + id + '/memory/target ' + str(val/1024))
self.xc.domain_set_target_mem(int(id), val/1024)
def mem_set(self, id, val):
uuid = self.domdict[id].uuid
print 'mem-set domain', id, 'to', val
self.xend_session.session.xenapi.VM.set_memory_dynamic_max_live(uuid, val)
self.xend_session.session.xenapi.VM.set_memory_dynamic_min_live(uuid, val)
def do_balloon(self, memsize):
MAX_TRIES = 20
niter = 0
prev_memory_actual = None
for i in self.domdict.keys():
self.domdict[i].no_progress = False
while True:
xenfree = self.get_free_xen_memory()
print 'got xenfree=', xenfree
if xenfree >= memsize:
return True
self.refresh_memactual()
if prev_memory_actual is not None:
for i in prev_memory_actual.keys():
if prev_memory_actual[i] == self.domdict[i].memory_actual:
self.domdict[i].no_progress = True
print 'domain', i, 'stuck at', self.domdict[i].memory_actual
memset_reqs = qmemman_algo.balloon(memsize-xenfree, self.domdict)
print 'requests:', memset_reqs
if niter > MAX_TRIES or len(memset_reqs) == 0:
return False
prev_memory_actual = {}
for i in memset_reqs:
dom, mem = i
self.mem_set(dom, mem)
prev_memory_actual[dom] = self.domdict[dom].memory_actual
time.sleep(self.BALOON_DELAY)
niter = niter + 1
def refresh_meminfo(self, domid, val):
self.domdict[domid].meminfo = self.parse_meminfo(val)
self.domdict[domid].meminfo_updated = True
def adjust_inflates_to_xenfree(self, reqs, idx):
i = idx
memory_needed = 0
while i < len(reqs):
dom, mem = reqs[i]
memory_needed += mem - self.domdict[dom].memory_actual
i = i + 1
scale = 1.0*self.get_free_xen_memory()/memory_needed
dom, mem = reqs[idx]
scaled_req = self.domdict[dom].memory_actual + scale*(mem - self.domdict[dom].memory_actual)
return int(scaled_req)
def do_balance(self):
if os.path.isfile('/etc/do-not-membalance'):
return
self.refresh_memactual()
xenfree = self.get_free_xen_memory()
memset_reqs = qmemman_algo.balance(xenfree, self.domdict)
wait_before_first_inflate = False
i = 0
while i < len(memset_reqs):
dom, mem = memset_reqs[i]
memory_change = mem - self.domdict[dom].memory_actual
if abs(memory_change) < 100*1024*1024:
i = i + 1
continue
if memory_change < 0:
wait_before_first_inflate = True
else:
if wait_before_first_inflate:
time.sleep(self.BALOON_DELAY)
wait_before_first_inflate = False
#the following is called before _each_ inflate, to account for possibility that
#previously triggered memory release is in progress
mem = self.adjust_inflates_to_xenfree(memset_reqs, i)
self.mem_set(dom, mem)
i = i + 1
# for i in self.domdict.keys():
# print 'domain ', i, ' meminfo=', self.domdict[i].meminfo, 'actual mem', self.domdict[i].memory_actual
# print 'domain ', i, 'actual mem', self.domdict[i].memory_actual
# print 'xen free mem', self.get_free_xen_memory()

101
dom0/qmemman/qmemman_algo.py Executable file
View File

@ -0,0 +1,101 @@
def is_suspicious(dom):
ret = False
if dom.meminfo['SwapTotal'] < dom.meminfo['SwapFree']:
ret = True
if dom.memory_actual < dom.meminfo['MemFree'] + dom.meminfo['Cached'] + dom.meminfo['Buffers']:
ret = True
if ret:
print 'suspicious meminfo for domain', dom.id, 'mem actual', dom.memory_actual, dom.meminfo
return ret
def recalc_mem_used(domdict):
for domid in domdict.keys():
dom = domdict[domid]
if dom.meminfo_updated:
dom.meminfo_updated = False
if is_suspicious(dom):
dom.meminfo = None
dom.mem_used = None
else:
dom.mem_used = dom.memory_actual - dom.meminfo['MemFree'] - dom.meminfo['Cached'] - dom.meminfo['Buffers'] + dom.meminfo['SwapTotal'] - dom.meminfo['SwapFree']
def prefmem(dom):
if dom.meminfo_updated:
raise AssertionError('meminfo_updated=True in prefmem')
CACHE_FACTOR = 1.3
#dom0 is special, as it must have large cache, for vbds. Thus, give it a special boost
if dom.id == '0':
return dom.mem_used*CACHE_FACTOR + 350*1024*1024
return dom.mem_used*CACHE_FACTOR
def memneeded(dom):
#do not change
#in balance(), "distribute totalsum proportionally to mempref" relies on this exact formula
ret = prefmem(dom) - dom.memory_actual
return ret
def balloon(memsize, domdict):
REQ_SAFETY_NET_FACTOR = 1.05
donors = list()
request = list()
available = 0
recalc_mem_used(domdict)
for i in domdict.keys():
if domdict[i].meminfo is None:
continue
if domdict[i].no_progress:
continue
need = memneeded(domdict[i])
if need < 0:
print 'balloon: dom' , i, 'has actual memory', domdict[i].memory_actual
donors.append((i,-need))
available-=need
print 'req=', memsize, 'avail=', available, 'donors', donors
if available<memsize:
return ()
scale = 1.0*memsize/available
for donors_iter in donors:
id, mem = donors_iter
memborrowed = mem*scale*REQ_SAFETY_NET_FACTOR
print 'borrow' , memborrowed, 'from', id
memtarget = int(domdict[id].memory_actual - memborrowed)
request.append((id, memtarget))
return request
# REQ_SAFETY_NET_FACTOR is a bit greater that 1. So that if the domain yields a bit less than requested, due
# to e.g. rounding errors, we will not get stuck. The surplus will return to the VM during "balance" call.
def balance(xenfree, domdict):
total_memneeded = 0
total_mem_pref = 0
recalc_mem_used(domdict)
#pass 1: compute the above "total" values
for i in domdict.keys():
if domdict[i].meminfo is None:
continue
need = memneeded(domdict[i])
print 'domain' , i, 'act/pref', domdict[i].memory_actual, prefmem(domdict[i]), 'need=', need
total_memneeded += need
total_mem_pref += prefmem(domdict[i])
totalsum = xenfree - total_memneeded
#pass 2: redistribute "totalsum" of memory between domains, proportionally to prefmem
donors = list()
acceptors = list()
for i in domdict.keys():
if domdict[i].meminfo is None:
continue
#distribute totalsum proportionally to mempref
scale = 1.0*prefmem(domdict[i])/total_mem_pref
target_nonint = prefmem(domdict[i]) + scale*totalsum
#prevent rounding errors
target = int(0.995*target_nonint)
if (target < domdict[i].memory_actual):
donors.append((i, target))
else:
acceptors.append((i, target))
print 'balance: xenfree=', xenfree, 'requests:', donors + acceptors
return donors + acceptors

16
dom0/qmemman/qmemman_client.py Executable file
View File

@ -0,0 +1,16 @@
import socket
class QMemmanClient:
def request_memory(self, amount):
self.sock = socket.socket(socket.AF_UNIX)
self.sock.connect("/var/run/qubes/qmemman.sock")
self.sock.send(str(amount)+"\n")
self.received = self.sock.recv(1024).strip()
if self.received == 'OK':
return True
else:
return False
def close(self):
self.sock.close()

123
dom0/qmemman/qmemman_server.py Executable file
View File

@ -0,0 +1,123 @@
#!/usr/bin/python
import SocketServer
import thread
import time
import xen.lowlevel.xs
import sys
import os
from qmemman import SystemState
system_state = SystemState()
global_lock = thread.allocate_lock()
additional_balance_delay = 0
def only_in_first_list(l1, l2):
ret=[]
for i in l1:
if not i in l2:
ret.append(i)
return ret
def get_req_node(domain_id):
return '/local/domain/'+domain_id+'/memory/meminfo'
class WatchType:
def __init__(self, fn, param):
self.fn = fn
self.param = param
class XS_Watcher:
def __init__(self):
self.handle = xen.lowlevel.xs.xs()
self.handle.watch('/local/domain', WatchType(XS_Watcher.dom_list_change, None))
self.watch_token_dict = {}
def dom_list_change(self, param):
curr = self.handle.ls('', '/local/domain')
if curr == None:
return
global_lock.acquire()
for i in only_in_first_list(curr, self.watch_token_dict.keys()):
watch = WatchType(XS_Watcher.request, i)
self.watch_token_dict[i] = watch
self.handle.watch(get_req_node(i), watch)
system_state.add_domain(i)
for i in only_in_first_list(self.watch_token_dict.keys(), curr):
self.handle.unwatch(get_req_node(i), self.watch_token_dict[i])
self.watch_token_dict.pop(i)
system_state.del_domain(i)
global_lock.release()
def request(self, domain_id):
ret = self.handle.read('', get_req_node(domain_id))
if ret == None or ret == '':
return
global_lock.acquire()
system_state.refresh_meminfo(domain_id, ret)
global_lock.release()
def watch_loop(self):
# sys.stderr = file('/var/log/qubes/qfileexchgd.errors', 'a')
while True:
result = self.handle.read_watch()
token = result[1]
token.fn(self, token.param)
class QMemmanReqHandler(SocketServer.BaseRequestHandler):
"""
The RequestHandler class for our server.
It is instantiated once per connection to the server, and must
override the handle() method to implement communication to the
client.
"""
def handle(self):
# self.request is the TCP socket connected to the client
while True:
self.data = self.request.recv(1024).strip()
if len(self.data) == 0:
print 'EOF'
return
if self.data == "DONE":
return
global_lock.acquire()
if system_state.do_balloon(int(self.data)):
resp = "OK\n"
additional_balance_delay = 5
else:
resp = "FAIL\n"
global_lock.release()
self.request.send(resp)
def start_server():
SOCK_PATH='/var/run/qubes/qmemman.sock'
try:
os.unlink(SOCK_PATH)
except:
pass
os.umask(0)
server = SocketServer.UnixStreamServer(SOCK_PATH, QMemmanReqHandler)
os.umask(077)
server.serve_forever()
def start_balancer():
while True:
time.sleep(1)
if additional_balance_delay == 0:
time.sleep(additional_balance_delay)
additional_balance_delay = 0
global_lock.acquire()
if additional_balance_delay == 0:
system_state.do_balance()
global_lock.release()
class QMemmanServer:
@staticmethod
def main():
thread.start_new_thread(start_server, tuple([]))
thread.start_new_thread(start_balancer, tuple([]))
XS_Watcher().watch_loop()

4
dom0/qmemman/server.py Executable file
View File

@ -0,0 +1,4 @@
#!/usr/bin/python
from qubes.qmemman_server import QMemmanServer
QMemmanServer.main()

29
dom0/qvm-core/qubes.py
View File

@ -29,6 +29,7 @@ import xml.parsers.expat
import fcntl
import re
import shutil
from qmemman_client import QMemmanClient
# Do not use XenAPI or create/read any VM files
# This is for testing only!
@ -326,6 +327,18 @@ class QubesVm(object):
return mem
def get_mem_dynamic_max(self):
if dry_run:
return 666
try:
mem = int(xend_session.session.xenapi.VM.get_memory_dynamic_max(self.session_uuid))
except XenAPI.Failure:
self.refresh_xend_session()
mem = int(xend_session.session.xenapi.VM.get_memory_dynamic_max(self.session_uuid))
return mem
def get_cpu_total_load(self):
if dry_run:
@ -474,15 +487,11 @@ class QubesVm(object):
if verbose:
print "--> Loading the VM (type = {0})...".format(self.type)
mem_required = self.get_mem_static_max()
dom0_mem = dom0_vm.get_mem()
dom0_mem_new = dom0_mem - mem_required + self.get_free_xen_memory()
if verbose:
print "--> AppVM required mem : {0}".format(mem_required)
print "--> Dom0 mem after launch : {0}".format(dom0_mem_new)
if dom0_mem_new < dom0_min_memory:
raise MemoryError ("ERROR: starting this VM would cause Dom0 memory to go below {0}B".format(dom0_min_memory))
mem_required = self.get_mem_dynamic_max()
qmemman_client = QMemmanClient()
if not qmemman_client.request_memory(mem_required):
qmemman_client.close()
raise MemoryError ("ERROR: insufficient memory to start this VM")
try:
xend_session.session.xenapi.VM.start (self.session_uuid, True) # Starting a VM paused
@ -490,6 +499,8 @@ class QubesVm(object):
self.refresh_xend_session()
xend_session.session.xenapi.VM.start (self.session_uuid, True) # Starting a VM paused
qmemman_client.close() # let qmemman_daemon resume balancing
xid = int (xend_session.session.xenapi.VM.get_domid (self.session_uuid))
if verbose:

2
rpm_spec/core-appvm.spec
View File

@ -65,6 +65,7 @@ cp qubes_timestamp qvm-copy-to-vm qvm-open-in-dvm $RPM_BUILD_ROOT/usr/bin
mkdir -p $RPM_BUILD_ROOT/usr/lib/qubes
cp qubes_add_pendrive_script qubes_penctl qvm-copy-to-vm.kde $RPM_BUILD_ROOT/usr/lib/qubes
ln -s /usr/bin/qvm-open-in-dvm $RPM_BUILD_ROOT/usr/lib/qubes/qvm-dvm-transfer
cp ../common/meminfo-writer $RPM_BUILD_ROOT/usr/lib/qubes
mkdir -p $RPM_BUILD_ROOT/%{kde_service_dir}
cp qvm-copy.desktop qvm-dvm.desktop $RPM_BUILD_ROOT/%{kde_service_dir}
mkdir -p $RPM_BUILD_ROOT/etc/udev/rules.d
@ -187,6 +188,7 @@ rm -rf $RPM_BUILD_ROOT
/usr/lib/qubes/qvm-copy-to-vm.kde
%attr(4755,root,root) /usr/bin/qvm-open-in-dvm
/usr/lib/qubes/qvm-dvm-transfer
/usr/lib/qubes/meminfo-writer
%{kde_service_dir}/qvm-copy.desktop
%{kde_service_dir}/qvm-dvm.desktop
%attr(4755,root,root) /usr/lib/qubes/qubes_penctl

11
rpm_spec/core-dom0.spec
View File

@ -44,8 +44,8 @@ Requires: python, xen-runtime, pciutils, python-inotify, python-daemon, kernel-q
The Qubes core files for installation on Dom0.
%build
python -m compileall qvm-core
python -O -m compileall qvm-core
python -m compileall qvm-core qmemman
python -O -m compileall qvm-core qmemman
make -C restore
%install
@ -67,6 +67,8 @@ cp qvm-core/qubes.py $RPM_BUILD_ROOT%{python_sitearch}/qubes
cp qvm-core/qubes.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes
cp qvm-core/__init__.py $RPM_BUILD_ROOT%{python_sitearch}/qubes
cp qvm-core/__init__.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes
cp qmemman/qmemman*py $RPM_BUILD_ROOT%{python_sitearch}/qubes
cp qmemman/qmemman*py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes
mkdir -p $RPM_BUILD_ROOT/usr/lib/qubes
cp aux-tools/patch_appvm_initramfs.sh $RPM_BUILD_ROOT/usr/lib/qubes
@ -77,6 +79,8 @@ cp aux-tools/convert_dirtemplate2vm.sh $RPM_BUILD_ROOT/usr/lib/qubes
cp aux-tools/create_apps_for_appvm.sh $RPM_BUILD_ROOT/usr/lib/qubes
cp aux-tools/remove_appvm_appmenus.sh $RPM_BUILD_ROOT/usr/lib/qubes
cp pendrive_swapper/qubes_pencmd $RPM_BUILD_ROOT/usr/lib/qubes
cp qmemman/server.py $RPM_BUILD_ROOT/usr/lib/qubes/qmemman_daemon.py
cp ../common/meminfo-writer $RPM_BUILD_ROOT/usr/lib/qubes/
cp restore/xenstore-watch restore/qvm-create-default-dvm $RPM_BUILD_ROOT/usr/bin
cp restore/qubes_restore restore/xenfreepages $RPM_BUILD_ROOT/usr/lib/qubes
@ -195,6 +199,7 @@ fi
%{python_sitearch}/qubes/__init__.py
%{python_sitearch}/qubes/__init__.pyc
%{python_sitearch}/qubes/__init__.pyo
%{python_sitearch}/qubes/qmemman*.py*
/usr/lib/qubes/patch_appvm_initramfs.sh
/usr/lib/qubes/unbind_pci_device.sh
/usr/lib/qubes/unbind_all_network_devices
@ -203,6 +208,8 @@ fi
/usr/lib/qubes/create_apps_for_appvm.sh
/usr/lib/qubes/remove_appvm_appmenus.sh
/usr/lib/qubes/qubes_pencmd
/usr/lib/qubes/qmemman_daemon.py*
/usr/lib/qubes/meminfo-writer
%attr(770,root,qubes) %dir /var/lib/qubes
%attr(770,root,qubes) %dir /var/lib/qubes/vm-templates
%attr(770,root,qubes) %dir /var/lib/qubes/appvms