From 62487c0f1e575434ba768508cfc943b3c275d987 Mon Sep 17 00:00:00 2001 From: Rafal Wojtczuk Date: Mon, 30 Aug 2010 11:40:19 +0200 Subject: [PATCH] Memory management across VMs, first release --- appvm/qubes_core | 1 + common/meminfo-writer | 4 + dom0/init.d/qubes_core | 2 + dom0/pendrive_swapper/qfilexchgd | 15 +-- dom0/qmemman/qmemman.py | 185 +++++++++++++++++++++++++++++++ dom0/qmemman/qmemman_algo.py | 101 +++++++++++++++++ dom0/qmemman/qmemman_client.py | 16 +++ dom0/qmemman/qmemman_server.py | 123 ++++++++++++++++++++ dom0/qmemman/server.py | 4 + dom0/qvm-core/qubes.py | 29 +++-- rpm_spec/core-appvm.spec | 2 + rpm_spec/core-dom0.spec | 11 +- 12 files changed, 475 insertions(+), 18 deletions(-) create mode 100755 common/meminfo-writer create mode 100755 dom0/qmemman/qmemman.py create mode 100755 dom0/qmemman/qmemman_algo.py create mode 100755 dom0/qmemman/qmemman_client.py create mode 100755 dom0/qmemman/qmemman_server.py create mode 100755 dom0/qmemman/server.py diff --git a/appvm/qubes_core b/appvm/qubes_core index c8dd1509..c291ec78 100755 --- a/appvm/qubes_core +++ b/appvm/qubes_core @@ -87,6 +87,7 @@ start() fi fi + /usr/lib/qubes/meminfo-writer & [ -x /rw/config/rc.local ] && /rw/config/rc.local success echo "" diff --git a/common/meminfo-writer b/common/meminfo-writer new file mode 100755 index 00000000..fdbfd29f --- /dev/null +++ b/common/meminfo-writer @@ -0,0 +1,4 @@ +#!/bin/sh +while sleep 1 ; do + xenstore-write memory/meminfo "`cat /proc/meminfo`" +done diff --git a/dom0/init.d/qubes_core b/dom0/init.d/qubes_core index 61452d76..e4bb5c3e 100755 --- a/dom0/init.d/qubes_core +++ b/dom0/init.d/qubes_core @@ -56,6 +56,8 @@ start() xm mem-set 0 1600 cp /var/lib/qubes/qubes.xml /var/lib/qubes/backup/qubes-$(date +%F-%T).xml setup_dvm_files + /usr/lib/qubes/qmemman_daemon.py >/var/log/qubes/qmemman.log 2>/var/log/qubes/qmemman.errs & + /usr/lib/qubes/meminfo-writer & touch /var/lock/subsys/qubes_core success echo diff --git a/dom0/pendrive_swapper/qfilexchgd b/dom0/pendrive_swapper/qfilexchgd index f08a235b..3773d0d2 100755 --- a/dom0/pendrive_swapper/qfilexchgd +++ b/dom0/pendrive_swapper/qfilexchgd @@ -30,6 +30,7 @@ import time from qubes.qubes import QubesVmCollection from qubes.qubes import QubesException from qubes.qubes import QubesDaemonPidfile +from qubes.qmemman_client import QMemmanClient filename_seq = 50 pen_cmd = '/usr/lib/qubes/qubes_pencmd' @@ -187,13 +188,11 @@ class DomainState: def handle_transfer_disposable(self, transaction_seq): - mem_for_dvm = 400 - xenfreepages_s = subprocess.Popen(["/usr/lib/qubes/xenfreepages"],stdout=subprocess.PIPE).stdout.readline() - xenfree_mb = int(xenfreepages_s)*4096/1024/1024 - if xenfree_mb < mem_for_dvm: - errmsg = 'Not enough memory to create DVM: ' - errmsg +='have ' + str(xenfree_mb) + 'MB, need ' - errmsg +=str(mem_for_dvm) + 'MB. Terminate some appVM and retry.' + qmemman_client = QMemmanClient() + if not qmemman_client.request_memory(400*1024*1024): + qmemman_client.close() + errmsg = 'Not enough memory to create DVM. ' + errmsg +='Terminate some appVM and retry.' subprocess.call(['/usr/bin/kdialog', '--sorry', errmsg]) return False @@ -205,12 +204,14 @@ class DomainState: if vm is None: logproc( 'Domain ' + vmname + ' does not exist ?') qvm_collection.unlock_db() + qmemman_client.close() return False retcode = subprocess.call(['/usr/lib/qubes/qubes_restore', current_savefile, '-c', vm.label.color, '-i', vm.label.icon, '-l', str(vm.label.index)]) + qmemman_client.close() if retcode != 0: subprocess.call(['/usr/bin/kdialog', '--sorry', 'DisposableVM creation failed, see qubes_restore.log']) qvm_collection.unlock_db() diff --git a/dom0/qmemman/qmemman.py b/dom0/qmemman/qmemman.py new file mode 100755 index 00000000..55e75e73 --- /dev/null +++ b/dom0/qmemman/qmemman.py @@ -0,0 +1,185 @@ +import xmlrpclib +from xen.xm import XenAPI +import xen.lowlevel.xc +import string +import time +import qmemman_algo +import os + +class XendSession(object): + def __init__(self): +# self.get_xend_session_old_api() + self.get_xend_session_new_api() + +# def get_xend_session_old_api(self): +# from xen.xend import XendClient +# from xen.util.xmlrpcclient import ServerProxy +# self.xend_server = ServerProxy(XendClient.uri) +# if self.xend_server is None: +# print "get_xend_session_old_api(): cannot open session!" + + + def get_xend_session_new_api(self): + xend_socket_uri = "httpu:///var/run/xend/xen-api.sock" + self.session = XenAPI.Session (xend_socket_uri) + self.session.login_with_password ("", "") + if self.session is None: + print "get_xend_session_new_api(): cannot open session!" + +class DomainState: + def __init__(self, id): + self.meminfo = None + self.memory_actual = None + self.mem_used = None + self.uuid = None + self.id = id + self.meminfo_updated = False + +class SystemState: + def __init__(self): + self.xend_session = XendSession() + self.domdict = {} + self.xc = xen.lowlevel.xc.xc() + self.BALOON_DELAY = 0.1 + + def add_domain(self, id): + self.domdict[id] = DomainState(id) + + def del_domain(self, id): + self.domdict.pop(id) + + def get_free_xen_memory(self): + return self.xc.physinfo()['free_memory']*1024 +# hosts = self.xend_session.session.xenapi.host.get_all() +# host_record = self.xend_session.session.xenapi.host.get_record(hosts[0]) +# host_metrics_record = self.xend_session.session.xenapi.host_metrics.get_record(host_record["metrics"]) +# ret = host_metrics_record["memory_free"] +# return long(ret) + + def refresh_memactual(self): + update_uuid_info = False + for domain in self.xc.domain_getinfo(): + id = str(domain['domid']) + if self.domdict.has_key(id): + self.domdict[id].memory_actual = domain['mem_kb']*1024 + if self.domdict[id].uuid is None: + update_uuid_info = True + if not update_uuid_info: + return + dom_recs = self.xend_session.session.xenapi.VM.get_all_records() +# dom_metrics_recs = self.xend_session.session.xenapi.VM_metrics.get_all_records() + for dom_ref, dom_rec in dom_recs.items(): +# dom_metrics_rec = dom_metrics_recs[dom_rec['metrics']] + id = dom_rec['domid'] +# mem = int(dom_metrics_rec['memory_actual'])/1024 + if (self.domdict.has_key(id)): +# self.domdict[id].memory_actual = mem + self.domdict[id].uuid = dom_rec['uuid'] + + def parse_meminfo(self, meminfo): + dict = {} + l1 = string.split(meminfo,"\n") + for i in l1: + l2 = string.split(i) + if len(l2) >= 2: + dict[string.rstrip(l2[0], ":")] = l2[1] + + try: + for i in ('MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree'): + val = int(dict[i])*1024 + if (val < 0): + return None + dict[i] = val + except: + return None + + if dict['SwapTotal'] < dict['SwapFree']: + return None + return dict + +#the below works (and is fast), but then 'xm list' shows unchanged memory value + def mem_set_alternative(self, id, val): + os.system('xenstore-write /local/domain/' + id + '/memory/target ' + str(val/1024)) + self.xc.domain_set_target_mem(int(id), val/1024) + + def mem_set(self, id, val): + uuid = self.domdict[id].uuid + print 'mem-set domain', id, 'to', val + self.xend_session.session.xenapi.VM.set_memory_dynamic_max_live(uuid, val) + self.xend_session.session.xenapi.VM.set_memory_dynamic_min_live(uuid, val) + + def do_balloon(self, memsize): + MAX_TRIES = 20 + niter = 0 + prev_memory_actual = None + for i in self.domdict.keys(): + self.domdict[i].no_progress = False + while True: + xenfree = self.get_free_xen_memory() + print 'got xenfree=', xenfree + if xenfree >= memsize: + return True + self.refresh_memactual() + if prev_memory_actual is not None: + for i in prev_memory_actual.keys(): + if prev_memory_actual[i] == self.domdict[i].memory_actual: + self.domdict[i].no_progress = True + print 'domain', i, 'stuck at', self.domdict[i].memory_actual + memset_reqs = qmemman_algo.balloon(memsize-xenfree, self.domdict) + print 'requests:', memset_reqs + if niter > MAX_TRIES or len(memset_reqs) == 0: + return False + prev_memory_actual = {} + for i in memset_reqs: + dom, mem = i + self.mem_set(dom, mem) + prev_memory_actual[dom] = self.domdict[dom].memory_actual + time.sleep(self.BALOON_DELAY) + niter = niter + 1 + + def refresh_meminfo(self, domid, val): + self.domdict[domid].meminfo = self.parse_meminfo(val) + self.domdict[domid].meminfo_updated = True + + def adjust_inflates_to_xenfree(self, reqs, idx): + i = idx + memory_needed = 0 + while i < len(reqs): + dom, mem = reqs[i] + memory_needed += mem - self.domdict[dom].memory_actual + i = i + 1 + scale = 1.0*self.get_free_xen_memory()/memory_needed + dom, mem = reqs[idx] + scaled_req = self.domdict[dom].memory_actual + scale*(mem - self.domdict[dom].memory_actual) + return int(scaled_req) + + def do_balance(self): + if os.path.isfile('/etc/do-not-membalance'): + return + self.refresh_memactual() + xenfree = self.get_free_xen_memory() + memset_reqs = qmemman_algo.balance(xenfree, self.domdict) + wait_before_first_inflate = False + i = 0 + while i < len(memset_reqs): + dom, mem = memset_reqs[i] + memory_change = mem - self.domdict[dom].memory_actual + if abs(memory_change) < 100*1024*1024: + i = i + 1 + continue + if memory_change < 0: + wait_before_first_inflate = True + else: + if wait_before_first_inflate: + time.sleep(self.BALOON_DELAY) + wait_before_first_inflate = False + #the following is called before _each_ inflate, to account for possibility that + #previously triggered memory release is in progress + mem = self.adjust_inflates_to_xenfree(memset_reqs, i) + self.mem_set(dom, mem) + i = i + 1 + +# for i in self.domdict.keys(): +# print 'domain ', i, ' meminfo=', self.domdict[i].meminfo, 'actual mem', self.domdict[i].memory_actual +# print 'domain ', i, 'actual mem', self.domdict[i].memory_actual +# print 'xen free mem', self.get_free_xen_memory() diff --git a/dom0/qmemman/qmemman_algo.py b/dom0/qmemman/qmemman_algo.py new file mode 100755 index 00000000..45cf6a59 --- /dev/null +++ b/dom0/qmemman/qmemman_algo.py @@ -0,0 +1,101 @@ +def is_suspicious(dom): + ret = False + if dom.meminfo['SwapTotal'] < dom.meminfo['SwapFree']: + ret = True + if dom.memory_actual < dom.meminfo['MemFree'] + dom.meminfo['Cached'] + dom.meminfo['Buffers']: + ret = True + if ret: + print 'suspicious meminfo for domain', dom.id, 'mem actual', dom.memory_actual, dom.meminfo + return ret + +def recalc_mem_used(domdict): + for domid in domdict.keys(): + dom = domdict[domid] + if dom.meminfo_updated: + dom.meminfo_updated = False + if is_suspicious(dom): + dom.meminfo = None + dom.mem_used = None + else: + dom.mem_used = dom.memory_actual - dom.meminfo['MemFree'] - dom.meminfo['Cached'] - dom.meminfo['Buffers'] + dom.meminfo['SwapTotal'] - dom.meminfo['SwapFree'] + +def prefmem(dom): + if dom.meminfo_updated: + raise AssertionError('meminfo_updated=True in prefmem') + CACHE_FACTOR = 1.3 +#dom0 is special, as it must have large cache, for vbds. Thus, give it a special boost + if dom.id == '0': + return dom.mem_used*CACHE_FACTOR + 350*1024*1024 + return dom.mem_used*CACHE_FACTOR + +def memneeded(dom): +#do not change +#in balance(), "distribute totalsum proportionally to mempref" relies on this exact formula + ret = prefmem(dom) - dom.memory_actual + return ret + + +def balloon(memsize, domdict): + REQ_SAFETY_NET_FACTOR = 1.05 + donors = list() + request = list() + available = 0 + recalc_mem_used(domdict) + for i in domdict.keys(): + if domdict[i].meminfo is None: + continue + if domdict[i].no_progress: + continue + need = memneeded(domdict[i]) + if need < 0: + print 'balloon: dom' , i, 'has actual memory', domdict[i].memory_actual + donors.append((i,-need)) + available-=need + print 'req=', memsize, 'avail=', available, 'donors', donors + if available Loading the VM (type = {0})...".format(self.type) - mem_required = self.get_mem_static_max() - dom0_mem = dom0_vm.get_mem() - dom0_mem_new = dom0_mem - mem_required + self.get_free_xen_memory() - if verbose: - print "--> AppVM required mem : {0}".format(mem_required) - print "--> Dom0 mem after launch : {0}".format(dom0_mem_new) - - if dom0_mem_new < dom0_min_memory: - raise MemoryError ("ERROR: starting this VM would cause Dom0 memory to go below {0}B".format(dom0_min_memory)) + mem_required = self.get_mem_dynamic_max() + qmemman_client = QMemmanClient() + if not qmemman_client.request_memory(mem_required): + qmemman_client.close() + raise MemoryError ("ERROR: insufficient memory to start this VM") try: xend_session.session.xenapi.VM.start (self.session_uuid, True) # Starting a VM paused @@ -490,6 +499,8 @@ class QubesVm(object): self.refresh_xend_session() xend_session.session.xenapi.VM.start (self.session_uuid, True) # Starting a VM paused + qmemman_client.close() # let qmemman_daemon resume balancing + xid = int (xend_session.session.xenapi.VM.get_domid (self.session_uuid)) if verbose: diff --git a/rpm_spec/core-appvm.spec b/rpm_spec/core-appvm.spec index a4444f0b..2949a059 100644 --- a/rpm_spec/core-appvm.spec +++ b/rpm_spec/core-appvm.spec @@ -65,6 +65,7 @@ cp qubes_timestamp qvm-copy-to-vm qvm-open-in-dvm $RPM_BUILD_ROOT/usr/bin mkdir -p $RPM_BUILD_ROOT/usr/lib/qubes cp qubes_add_pendrive_script qubes_penctl qvm-copy-to-vm.kde $RPM_BUILD_ROOT/usr/lib/qubes ln -s /usr/bin/qvm-open-in-dvm $RPM_BUILD_ROOT/usr/lib/qubes/qvm-dvm-transfer +cp ../common/meminfo-writer $RPM_BUILD_ROOT/usr/lib/qubes mkdir -p $RPM_BUILD_ROOT/%{kde_service_dir} cp qvm-copy.desktop qvm-dvm.desktop $RPM_BUILD_ROOT/%{kde_service_dir} mkdir -p $RPM_BUILD_ROOT/etc/udev/rules.d @@ -187,6 +188,7 @@ rm -rf $RPM_BUILD_ROOT /usr/lib/qubes/qvm-copy-to-vm.kde %attr(4755,root,root) /usr/bin/qvm-open-in-dvm /usr/lib/qubes/qvm-dvm-transfer +/usr/lib/qubes/meminfo-writer %{kde_service_dir}/qvm-copy.desktop %{kde_service_dir}/qvm-dvm.desktop %attr(4755,root,root) /usr/lib/qubes/qubes_penctl diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index 456d959c..f4af95d8 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -44,8 +44,8 @@ Requires: python, xen-runtime, pciutils, python-inotify, python-daemon, kernel-q The Qubes core files for installation on Dom0. %build -python -m compileall qvm-core -python -O -m compileall qvm-core +python -m compileall qvm-core qmemman +python -O -m compileall qvm-core qmemman make -C restore %install @@ -67,6 +67,8 @@ cp qvm-core/qubes.py $RPM_BUILD_ROOT%{python_sitearch}/qubes cp qvm-core/qubes.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes cp qvm-core/__init__.py $RPM_BUILD_ROOT%{python_sitearch}/qubes cp qvm-core/__init__.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes +cp qmemman/qmemman*py $RPM_BUILD_ROOT%{python_sitearch}/qubes +cp qmemman/qmemman*py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes mkdir -p $RPM_BUILD_ROOT/usr/lib/qubes cp aux-tools/patch_appvm_initramfs.sh $RPM_BUILD_ROOT/usr/lib/qubes @@ -77,6 +79,8 @@ cp aux-tools/convert_dirtemplate2vm.sh $RPM_BUILD_ROOT/usr/lib/qubes cp aux-tools/create_apps_for_appvm.sh $RPM_BUILD_ROOT/usr/lib/qubes cp aux-tools/remove_appvm_appmenus.sh $RPM_BUILD_ROOT/usr/lib/qubes cp pendrive_swapper/qubes_pencmd $RPM_BUILD_ROOT/usr/lib/qubes +cp qmemman/server.py $RPM_BUILD_ROOT/usr/lib/qubes/qmemman_daemon.py +cp ../common/meminfo-writer $RPM_BUILD_ROOT/usr/lib/qubes/ cp restore/xenstore-watch restore/qvm-create-default-dvm $RPM_BUILD_ROOT/usr/bin cp restore/qubes_restore restore/xenfreepages $RPM_BUILD_ROOT/usr/lib/qubes @@ -195,6 +199,7 @@ fi %{python_sitearch}/qubes/__init__.py %{python_sitearch}/qubes/__init__.pyc %{python_sitearch}/qubes/__init__.pyo +%{python_sitearch}/qubes/qmemman*.py* /usr/lib/qubes/patch_appvm_initramfs.sh /usr/lib/qubes/unbind_pci_device.sh /usr/lib/qubes/unbind_all_network_devices @@ -203,6 +208,8 @@ fi /usr/lib/qubes/create_apps_for_appvm.sh /usr/lib/qubes/remove_appvm_appmenus.sh /usr/lib/qubes/qubes_pencmd +/usr/lib/qubes/qmemman_daemon.py* +/usr/lib/qubes/meminfo-writer %attr(770,root,qubes) %dir /var/lib/qubes %attr(770,root,qubes) %dir /var/lib/qubes/vm-templates %attr(770,root,qubes) %dir /var/lib/qubes/appvms