qmemman.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317
  1. #!/usr/bin/python2
  2. # -*- coding: utf-8 -*-
  3. #
  4. # The Qubes OS Project, http://www.qubes-os.org
  5. #
  6. # Copyright (C) 2010 Rafal Wojtczuk <rafal@invisiblethingslab.com>
  7. # Copyright (C) 2013 Marek Marczykowski <marmarek@invisiblethingslab.com>
  8. #
  9. # This program is free software; you can redistribute it and/or
  10. # modify it under the terms of the GNU General Public License
  11. # as published by the Free Software Foundation; either version 2
  12. # of the License, or (at your option) any later version.
  13. #
  14. # This program is distributed in the hope that it will be useful,
  15. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. # GNU General Public License for more details.
  18. #
  19. # You should have received a copy of the GNU General Public License
  20. # along with this program; if not, write to the Free Software
  21. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  22. #
  23. #
  24. import xen.lowlevel.xc
  25. import xen.lowlevel.xs
  26. import string
  27. import time
  28. import qmemman_algo
  29. import os
  30. from notify import notify_error_qubes_manager, clear_error_qubes_manager
  31. import logging
  32. no_progress_msg="VM refused to give back requested memory"
  33. slow_memset_react_msg="VM didn't give back all requested memory"
  34. class DomainState:
  35. def __init__(self, id):
  36. self.meminfo = None #dictionary of memory info read from client
  37. self.memory_actual = None #the current memory size
  38. self.memory_maximum = None #the maximum memory size
  39. self.mem_used = None #used memory, computed based on meminfo
  40. self.id = id #domain id
  41. self.last_target = 0 #the last memset target
  42. self.no_progress = False #no react to memset
  43. self.slow_memset_react = False #slow react to memset (after few tries still above target)
  44. class SystemState(object):
  45. def __init__(self):
  46. self.log = logging.getLogger('qmemman.systemstate')
  47. self.log.debug('SystemState()')
  48. self.domdict = {}
  49. self.xc = xen.lowlevel.xc.xc()
  50. self.xs = xen.lowlevel.xs.xs()
  51. self.BALOON_DELAY = 0.1
  52. self.XEN_FREE_MEM_LEFT = 50*1024*1024
  53. self.XEN_FREE_MEM_MIN = 25*1024*1024
  54. # Overhead of per-page Xen structures, taken from OpenStack nova/virt/xenapi/driver.py
  55. # see https://wiki.openstack.org/wiki/XenServer/Overhead
  56. # we divide total and free physical memory by this to get "assignable" memory
  57. self.MEM_OVERHEAD_FACTOR = 1.0 / 1.00781
  58. self.ALL_PHYS_MEM = int(self.xc.physinfo()['total_memory']*1024 * self.MEM_OVERHEAD_FACTOR)
  59. def add_domain(self, id):
  60. self.log.debug('add_domain(id={!r})'.format(id))
  61. self.domdict[id] = DomainState(id)
  62. def del_domain(self, id):
  63. self.log.debug('del_domain(id={!r})'.format(id))
  64. self.domdict.pop(id)
  65. def get_free_xen_memory(self):
  66. return int(self.xc.physinfo()['free_memory']*1024 * self.MEM_OVERHEAD_FACTOR)
  67. # hosts = self.xend_session.session.xenapi.host.get_all()
  68. # host_record = self.xend_session.session.xenapi.host.get_record(hosts[0])
  69. # host_metrics_record = self.xend_session.session.xenapi.host_metrics.get_record(host_record["metrics"])
  70. # ret = host_metrics_record["memory_free"]
  71. # return long(ret)
  72. #refresh information on memory assigned to all domains
  73. def refresh_memactual(self):
  74. for domain in self.xc.domain_getinfo():
  75. id = str(domain['domid'])
  76. if self.domdict.has_key(id):
  77. self.domdict[id].memory_actual = domain['mem_kb']*1024
  78. self.domdict[id].memory_maximum = self.xs.read('', '/local/domain/%s/memory/static-max' % str(id))
  79. if self.domdict[id].memory_maximum:
  80. self.domdict[id].memory_maximum = int(self.domdict[id].memory_maximum)*1024
  81. else:
  82. self.domdict[id].memory_maximum = self.ALL_PHYS_MEM
  83. # the previous line used to be
  84. # self.domdict[id].memory_maximum = domain['maxmem_kb']*1024
  85. # but domain['maxmem_kb'] changes in self.mem_set as well, and this results in
  86. # the memory never increasing
  87. # in fact, the only possible case of nonexisting memory/static-max is dom0
  88. # see #307
  89. def clear_outdated_error_markers(self):
  90. # Clear outdated errors
  91. for i in self.domdict.keys():
  92. if self.domdict[i].slow_memset_react and \
  93. self.domdict[i].memory_actual <= self.domdict[i].last_target + self.XEN_FREE_MEM_LEFT/4:
  94. dom_name = self.xs.read('', '/local/domain/%s/name' % str(i))
  95. if dom_name is not None:
  96. clear_error_qubes_manager(dom_name, slow_memset_react_msg)
  97. self.domdict[i].slow_memset_react = False
  98. if self.domdict[i].no_progress and \
  99. self.domdict[i].memory_actual <= self.domdict[i].last_target + self.XEN_FREE_MEM_LEFT/4:
  100. dom_name = self.xs.read('', '/local/domain/%s/name' % str(i))
  101. if dom_name is not None:
  102. clear_error_qubes_manager(dom_name, no_progress_msg)
  103. self.domdict[i].no_progress = False
  104. #the below works (and is fast), but then 'xm list' shows unchanged memory value
  105. def mem_set(self, id, val):
  106. self.log.info('mem-set domain {} to {}'.format(id, val))
  107. self.domdict[id].last_target = val
  108. #can happen in the middle of domain shutdown
  109. #apparently xc.lowlevel throws exceptions too
  110. try:
  111. self.xc.domain_setmaxmem(int(id), int(val/1024) + 1024) # LIBXL_MAXMEM_CONSTANT=1024
  112. self.xc.domain_set_target_mem(int(id), int(val/1024))
  113. except:
  114. pass
  115. self.xs.write('', '/local/domain/' + id + '/memory/target', str(int(val/1024)))
  116. # this is called at the end of ballooning, when we have Xen free mem already
  117. # make sure that past mem_set will not decrease Xen free mem
  118. def inhibit_balloon_up(self):
  119. self.log.debug('inhibit_balloon_up()')
  120. for i in self.domdict.keys():
  121. dom = self.domdict[i]
  122. if dom.memory_actual is not None and dom.memory_actual + 200*1024 < dom.last_target:
  123. self.log.info(
  124. 'Preventing balloon up to {}'.format(dom.last_target))
  125. self.mem_set(i, dom.memory_actual)
  126. #perform memory ballooning, across all domains, to add "memsize" to Xen free memory
  127. def do_balloon(self, memsize):
  128. self.log.info('do_balloon(memsize={!r})'.format(memsize))
  129. CHECK_PERIOD_S = 3
  130. CHECK_MB_S = 100
  131. niter = 0
  132. prev_memory_actual = None
  133. for i in self.domdict.keys():
  134. self.domdict[i].no_progress = False
  135. #: number of loop iterations for CHECK_PERIOD_S seconds
  136. check_period = max(1, int((CHECK_PERIOD_S + 0.0) / self.BALOON_DELAY))
  137. #: number of free memory bytes expected to get during CHECK_PERIOD_S
  138. #: seconds
  139. check_delta = CHECK_PERIOD_S * CHECK_MB_S * 1024 * 1024
  140. #: helper array for holding free memory size, CHECK_PERIOD_S seconds
  141. #: ago, at every loop iteration
  142. xenfree_ring = [0] * check_period
  143. while True:
  144. self.log.debug('niter={:2d}'.format(niter))
  145. self.refresh_memactual()
  146. xenfree = self.get_free_xen_memory()
  147. self.log.info('xenfree={!r}'.format(xenfree))
  148. if xenfree >= memsize + self.XEN_FREE_MEM_MIN:
  149. self.inhibit_balloon_up()
  150. return True
  151. # fail the request if over past CHECK_PERIOD_S seconds,
  152. # we got less than CHECK_MB_S MB/s on average
  153. ring_slot = niter % check_period
  154. if niter >= check_period and xenfree < xenfree_ring[ring_slot] + check_delta:
  155. return False
  156. xenfree_ring[ring_slot] = xenfree
  157. if prev_memory_actual is not None:
  158. for i in prev_memory_actual.keys():
  159. if prev_memory_actual[i] == self.domdict[i].memory_actual:
  160. #domain not responding to memset requests, remove it from donors
  161. self.domdict[i].no_progress = True
  162. self.log.info('domain {} stuck at {}'.format(i, self.domdict[i].memory_actual))
  163. memset_reqs = qmemman_algo.balloon(memsize + self.XEN_FREE_MEM_LEFT - xenfree, self.domdict)
  164. self.log.info('memset_reqs={!r}'.format(memset_reqs))
  165. if len(memset_reqs) == 0:
  166. return False
  167. prev_memory_actual = {}
  168. for i in memset_reqs:
  169. dom, mem = i
  170. self.mem_set(dom, mem)
  171. prev_memory_actual[dom] = self.domdict[dom].memory_actual
  172. self.log.debug('sleeping for {} s'.format(self.BALOON_DELAY))
  173. time.sleep(self.BALOON_DELAY)
  174. niter = niter + 1
  175. def refresh_meminfo(self, domid, untrusted_meminfo_key):
  176. self.log.debug(
  177. 'refresh_meminfo(domid={}, untrusted_meminfo_key={!r})'.format(
  178. domid, untrusted_meminfo_key))
  179. qmemman_algo.refresh_meminfo_for_domain(
  180. self.domdict[domid], untrusted_meminfo_key)
  181. self.do_balance()
  182. #is the computed balance request big enough ?
  183. #so that we do not trash with small adjustments
  184. def is_balance_req_significant(self, memset_reqs, xenfree):
  185. self.log.debug(
  186. 'is_balance_req_significant(memset_reqs={}, xenfree={})'.format(
  187. memset_reqs, xenfree))
  188. total_memory_transfer = 0
  189. MIN_TOTAL_MEMORY_TRANSFER = 150*1024*1024
  190. MIN_MEM_CHANGE_WHEN_UNDER_PREF = 15*1024*1024
  191. # If xenfree to low, return immediately
  192. if self.XEN_FREE_MEM_LEFT - xenfree > MIN_MEM_CHANGE_WHEN_UNDER_PREF:
  193. self.log.debug('xenfree is too low, returning')
  194. return True
  195. for rq in memset_reqs:
  196. dom, mem = rq
  197. last_target = self.domdict[dom].last_target
  198. memory_change = mem - last_target
  199. total_memory_transfer += abs(memory_change)
  200. pref = qmemman_algo.prefmem(self.domdict[dom])
  201. if last_target > 0 and last_target < pref and memory_change > MIN_MEM_CHANGE_WHEN_UNDER_PREF:
  202. self.log.info(
  203. 'dom {} is below pref, allowing balance'.format(dom))
  204. return True
  205. ret = total_memory_transfer + abs(xenfree - self.XEN_FREE_MEM_LEFT) > MIN_TOTAL_MEMORY_TRANSFER
  206. self.log.debug('is_balance_req_significant return {}'.format(ret))
  207. return ret
  208. def print_stats(self, xenfree, memset_reqs):
  209. for i in self.domdict.keys():
  210. if self.domdict[i].meminfo is not None:
  211. self.log.info('stat: dom {!r} act={} pref={}'.format(i,
  212. self.domdict[i].memory_actual,
  213. qmemman_algo.prefmem(self.domdict[i])))
  214. self.log.info('stat: xenfree={} memset_reqs={}'.format(xenfree, memset_reqs))
  215. def do_balance(self):
  216. self.log.debug('do_balance()')
  217. if os.path.isfile('/var/run/qubes/do-not-membalance'):
  218. self.log.debug('do-not-membalance file preset, returning')
  219. return
  220. self.refresh_memactual()
  221. self.clear_outdated_error_markers()
  222. xenfree = self.get_free_xen_memory()
  223. memset_reqs = qmemman_algo.balance(xenfree - self.XEN_FREE_MEM_LEFT, self.domdict)
  224. if not self.is_balance_req_significant(memset_reqs, xenfree):
  225. return
  226. self.print_stats(xenfree, memset_reqs)
  227. prev_memactual = {}
  228. for i in self.domdict.keys():
  229. prev_memactual[i] = self.domdict[i].memory_actual
  230. for rq in memset_reqs:
  231. dom, mem = rq
  232. # Force to always have at least 0.9*self.XEN_FREE_MEM_LEFT (some
  233. # margin for rounding errors). Before giving memory to
  234. # domain, ensure that others have gived it back.
  235. # If not - wait a little.
  236. ntries = 5
  237. while self.get_free_xen_memory() - (mem - self.domdict[dom].memory_actual) < 0.9*self.XEN_FREE_MEM_LEFT:
  238. self.log.debug('do_balance dom={!r} sleeping ntries={}'.format(
  239. dom, ntries))
  240. time.sleep(self.BALOON_DELAY)
  241. ntries -= 1
  242. if ntries <= 0:
  243. # Waiting haven't helped; Find which domain get stuck and
  244. # abort balance (after distributing what we have)
  245. self.refresh_memactual()
  246. for rq2 in memset_reqs:
  247. dom2, mem2 = rq2
  248. if dom2 == dom:
  249. # All donors have been procesed
  250. break
  251. # allow some small margin
  252. if self.domdict[dom2].memory_actual > self.domdict[dom2].last_target + self.XEN_FREE_MEM_LEFT/4:
  253. # VM didn't react to memory request at all, remove from donors
  254. if prev_memactual[dom2] == self.domdict[dom2].memory_actual:
  255. self.log.warning(
  256. 'dom {!r} didnt react to memory request'
  257. ' (holds {}, requested balloon down to {})'
  258. .format(dom2,
  259. self.domdict[dom2].memory_actual,
  260. mem2))
  261. self.domdict[dom2].no_progress = True
  262. dom_name = self.xs.read('', '/local/domain/%s/name' % str(dom2))
  263. if dom_name is not None:
  264. notify_error_qubes_manager(str(dom_name), no_progress_msg)
  265. else:
  266. self.log.warning('dom {!r} still hold more'
  267. ' memory than have assigned ({} > {})'
  268. .format(dom2,
  269. self.domdict[dom2].memory_actual,
  270. mem2))
  271. self.domdict[dom2].slow_memset_react = True
  272. dom_name = self.xs.read('', '/local/domain/%s/name' % str(dom2))
  273. if dom_name is not None:
  274. notify_error_qubes_manager(str(dom_name), slow_memset_react_msg)
  275. self.mem_set(dom, self.get_free_xen_memory() + self.domdict[dom].memory_actual - self.XEN_FREE_MEM_LEFT)
  276. return
  277. self.mem_set(dom, mem)
  278. # for i in self.domdict.keys():
  279. # print 'domain ', i, ' meminfo=', self.domdict[i].meminfo, 'actual mem', self.domdict[i].memory_actual
  280. # print 'domain ', i, 'actual mem', self.domdict[i].memory_actual
  281. # print 'xen free mem', self.get_free_xen_memory()