qmemman.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296
  1. #!/usr/bin/python2
  2. # -*- coding: utf-8 -*-
  3. #
  4. # The Qubes OS Project, http://www.qubes-os.org
  5. #
  6. # Copyright (C) 2010 Rafal Wojtczuk <rafal@invisiblethingslab.com>
  7. # Copyright (C) 2013 Marek Marczykowski <marmarek@invisiblethingslab.com>
  8. #
  9. # This program is free software; you can redistribute it and/or
  10. # modify it under the terms of the GNU General Public License
  11. # as published by the Free Software Foundation; either version 2
  12. # of the License, or (at your option) any later version.
  13. #
  14. # This program is distributed in the hope that it will be useful,
  15. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. # GNU General Public License for more details.
  18. #
  19. # You should have received a copy of the GNU General Public License
  20. # along with this program; if not, write to the Free Software
  21. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  22. #
  23. #
  24. import xen.lowlevel.xc
  25. import xen.lowlevel.xs
  26. import string
  27. import time
  28. import qmemman_algo
  29. import os
  30. from notify import notify_error_qubes_manager, clear_error_qubes_manager
  31. import logging
  32. no_progress_msg="VM refused to give back requested memory"
  33. slow_memset_react_msg="VM didn't give back all requested memory"
  34. class DomainState:
  35. def __init__(self, id):
  36. self.meminfo = None #dictionary of memory info read from client
  37. self.memory_actual = None #the current memory size
  38. self.memory_maximum = None #the maximum memory size
  39. self.mem_used = None #used memory, computed based on meminfo
  40. self.id = id #domain id
  41. self.last_target = 0 #the last memset target
  42. self.no_progress = False #no react to memset
  43. self.slow_memset_react = False #slow react to memset (after few tries still above target)
  44. class SystemState(object):
  45. def __init__(self):
  46. self.log = logging.getLogger('qmemman.systemstate')
  47. self.log.debug('SystemState()')
  48. self.domdict = {}
  49. self.xc = xen.lowlevel.xc.xc()
  50. self.xs = xen.lowlevel.xs.xs()
  51. self.BALOON_DELAY = 0.1
  52. self.XEN_FREE_MEM_LEFT = 50*1024*1024
  53. self.XEN_FREE_MEM_MIN = 25*1024*1024
  54. self.ALL_PHYS_MEM = self.xc.physinfo()['total_memory']*1024
  55. def add_domain(self, id):
  56. self.log.debug('add_domain(id={!r})'.format(id))
  57. self.domdict[id] = DomainState(id)
  58. def del_domain(self, id):
  59. self.log.debug('del_domain(id={!r})'.format(id))
  60. self.domdict.pop(id)
  61. def get_free_xen_memory(self):
  62. return self.xc.physinfo()['free_memory']*1024
  63. # hosts = self.xend_session.session.xenapi.host.get_all()
  64. # host_record = self.xend_session.session.xenapi.host.get_record(hosts[0])
  65. # host_metrics_record = self.xend_session.session.xenapi.host_metrics.get_record(host_record["metrics"])
  66. # ret = host_metrics_record["memory_free"]
  67. # return long(ret)
  68. #refresh information on memory assigned to all domains
  69. def refresh_memactual(self):
  70. for domain in self.xc.domain_getinfo():
  71. id = str(domain['domid'])
  72. if self.domdict.has_key(id):
  73. self.domdict[id].memory_actual = domain['mem_kb']*1024
  74. self.domdict[id].memory_maximum = self.xs.read('', '/local/domain/%s/memory/static-max' % str(id))
  75. if self.domdict[id].memory_maximum:
  76. self.domdict[id].memory_maximum = int(self.domdict[id].memory_maximum)*1024
  77. else:
  78. self.domdict[id].memory_maximum = self.ALL_PHYS_MEM
  79. # the previous line used to be
  80. # self.domdict[id].memory_maximum = domain['maxmem_kb']*1024
  81. # but domain['maxmem_kb'] changes in self.mem_set as well, and this results in
  82. # the memory never increasing
  83. # in fact, the only possible case of nonexisting memory/static-max is dom0
  84. # see #307
  85. def clear_outdated_error_markers(self):
  86. # Clear outdated errors
  87. for i in self.domdict.keys():
  88. if self.domdict[i].slow_memset_react and \
  89. self.domdict[i].memory_actual <= self.domdict[i].last_target + self.XEN_FREE_MEM_LEFT/4:
  90. dom_name = self.xs.read('', '/local/domain/%s/name' % str(i))
  91. if dom_name is not None:
  92. clear_error_qubes_manager(dom_name, slow_memset_react_msg)
  93. self.domdict[i].slow_memset_react = False
  94. if self.domdict[i].no_progress and \
  95. self.domdict[i].memory_actual <= self.domdict[i].last_target + self.XEN_FREE_MEM_LEFT/4:
  96. dom_name = self.xs.read('', '/local/domain/%s/name' % str(i))
  97. if dom_name is not None:
  98. clear_error_qubes_manager(dom_name, no_progress_msg)
  99. self.domdict[i].no_progress = False
  100. #the below works (and is fast), but then 'xm list' shows unchanged memory value
  101. def mem_set(self, id, val):
  102. self.log.info('mem-set domain {} to {}'.format(id, val))
  103. self.domdict[id].last_target = val
  104. #can happen in the middle of domain shutdown
  105. #apparently xc.lowlevel throws exceptions too
  106. try:
  107. self.xc.domain_setmaxmem(int(id), int(val/1024) + 1024) # LIBXL_MAXMEM_CONSTANT=1024
  108. self.xc.domain_set_target_mem(int(id), int(val/1024))
  109. except:
  110. pass
  111. self.xs.write('', '/local/domain/' + id + '/memory/target', str(int(val/1024)))
  112. # this is called at the end of ballooning, when we have Xen free mem already
  113. # make sure that past mem_set will not decrease Xen free mem
  114. def inhibit_balloon_up(self):
  115. self.log.debug('inhibit_balloon_up()')
  116. for i in self.domdict.keys():
  117. dom = self.domdict[i]
  118. if dom.memory_actual is not None and dom.memory_actual + 200*1024 < dom.last_target:
  119. self.log.info(
  120. 'Preventing balloon up to {}'.format(dom.last_target))
  121. self.mem_set(i, dom.memory_actual)
  122. #perform memory ballooning, across all domains, to add "memsize" to Xen free memory
  123. def do_balloon(self, memsize):
  124. self.log.info('do_balloon(memsize={!r})'.format(memsize))
  125. MAX_TRIES = 20
  126. niter = 0
  127. prev_memory_actual = None
  128. for i in self.domdict.keys():
  129. self.domdict[i].no_progress = False
  130. while True:
  131. self.log.debug('niter={:2d}/{:2d}'.format(niter, MAX_TRIES))
  132. self.refresh_memactual()
  133. xenfree = self.get_free_xen_memory()
  134. self.log.info('xenfree={!r}'.format(xenfree))
  135. if xenfree >= memsize + self.XEN_FREE_MEM_MIN:
  136. self.inhibit_balloon_up()
  137. return True
  138. if prev_memory_actual is not None:
  139. for i in prev_memory_actual.keys():
  140. if prev_memory_actual[i] == self.domdict[i].memory_actual:
  141. #domain not responding to memset requests, remove it from donors
  142. self.domdict[i].no_progress = True
  143. self.log.info('domain {} stuck at {}'.format(i, self.domdict[i].memory_actual))
  144. memset_reqs = qmemman_algo.balloon(memsize + self.XEN_FREE_MEM_LEFT - xenfree, self.domdict)
  145. self.log.info('memset_reqs={!r}'.format(memset_reqs))
  146. if niter > MAX_TRIES or len(memset_reqs) == 0:
  147. return False
  148. prev_memory_actual = {}
  149. for i in memset_reqs:
  150. dom, mem = i
  151. self.mem_set(dom, mem)
  152. prev_memory_actual[dom] = self.domdict[dom].memory_actual
  153. self.log.debug('sleeping for {} s'.format(self.BALOON_DELAY))
  154. time.sleep(self.BALOON_DELAY)
  155. niter = niter + 1
  156. def refresh_meminfo(self, domid, untrusted_meminfo_key):
  157. self.log.debug(
  158. 'refresh_meminfo(domid={}, untrusted_meminfo_key={!r})'.format(
  159. domid, untrusted_meminfo_key))
  160. qmemman_algo.refresh_meminfo_for_domain(
  161. self.domdict[domid], untrusted_meminfo_key)
  162. self.do_balance()
  163. #is the computed balance request big enough ?
  164. #so that we do not trash with small adjustments
  165. def is_balance_req_significant(self, memset_reqs, xenfree):
  166. self.log.debug(
  167. 'is_balance_req_significant(memset_reqs={}, xenfree={})'.format(
  168. memset_reqs, xenfree))
  169. total_memory_transfer = 0
  170. MIN_TOTAL_MEMORY_TRANSFER = 150*1024*1024
  171. MIN_MEM_CHANGE_WHEN_UNDER_PREF = 15*1024*1024
  172. # If xenfree to low, return immediately
  173. if self.XEN_FREE_MEM_LEFT - xenfree > MIN_MEM_CHANGE_WHEN_UNDER_PREF:
  174. self.log.debug('xenfree is too low, returning')
  175. return True
  176. for rq in memset_reqs:
  177. dom, mem = rq
  178. last_target = self.domdict[dom].last_target
  179. memory_change = mem - last_target
  180. total_memory_transfer += abs(memory_change)
  181. pref = qmemman_algo.prefmem(self.domdict[dom])
  182. if last_target > 0 and last_target < pref and memory_change > MIN_MEM_CHANGE_WHEN_UNDER_PREF:
  183. self.log.info(
  184. 'dom {} is below pref, allowing balance'.format(dom))
  185. return True
  186. ret = total_memory_transfer + abs(xenfree - self.XEN_FREE_MEM_LEFT) > MIN_TOTAL_MEMORY_TRANSFER
  187. self.log.debug('is_balance_req_significant return {}'.format(ret))
  188. return ret
  189. def print_stats(self, xenfree, memset_reqs):
  190. for i in self.domdict.keys():
  191. if self.domdict[i].meminfo is not None:
  192. self.log.info('stat: dom {!r} act={} pref={}'.format(i,
  193. self.domdict[i].memory_actual,
  194. qmemman_algo.prefmem(self.domdict[i])))
  195. self.log.info('stat: xenfree={} memset_reqs={}'.format(xenfree, memset_reqs))
  196. def do_balance(self):
  197. self.log.debug('do_balance()')
  198. if os.path.isfile('/var/run/qubes/do-not-membalance'):
  199. self.log.debug('do-not-membalance file preset, returning')
  200. return
  201. self.refresh_memactual()
  202. self.clear_outdated_error_markers()
  203. xenfree = self.get_free_xen_memory()
  204. memset_reqs = qmemman_algo.balance(xenfree - self.XEN_FREE_MEM_LEFT, self.domdict)
  205. if not self.is_balance_req_significant(memset_reqs, xenfree):
  206. return
  207. self.print_stats(xenfree, memset_reqs)
  208. prev_memactual = {}
  209. for i in self.domdict.keys():
  210. prev_memactual[i] = self.domdict[i].memory_actual
  211. for rq in memset_reqs:
  212. dom, mem = rq
  213. # Force to always have at least 0.9*self.XEN_FREE_MEM_LEFT (some
  214. # margin for rounding errors). Before giving memory to
  215. # domain, ensure that others have gived it back.
  216. # If not - wait a little.
  217. ntries = 5
  218. while self.get_free_xen_memory() - (mem - self.domdict[dom].memory_actual) < 0.9*self.XEN_FREE_MEM_LEFT:
  219. self.log.debug('do_balance dom={!r} sleeping ntries={}'.format(
  220. dom, ntries))
  221. time.sleep(self.BALOON_DELAY)
  222. ntries -= 1
  223. if ntries <= 0:
  224. # Waiting haven't helped; Find which domain get stuck and
  225. # abort balance (after distributing what we have)
  226. self.refresh_memactual()
  227. for rq2 in memset_reqs:
  228. dom2, mem2 = rq2
  229. if dom2 == dom:
  230. # All donors have been procesed
  231. break
  232. # allow some small margin
  233. if self.domdict[dom2].memory_actual > self.domdict[dom2].last_target + self.XEN_FREE_MEM_LEFT/4:
  234. # VM didn't react to memory request at all, remove from donors
  235. if prev_memactual[dom2] == self.domdict[dom2].memory_actual:
  236. self.log.warning(
  237. 'dom {!r} didnt react to memory request'
  238. ' (holds {}, requested balloon down to {})'
  239. .format(dom2,
  240. self.domdict[dom2].memory_actual,
  241. mem2))
  242. self.domdict[dom2].no_progress = True
  243. dom_name = self.xs.read('', '/local/domain/%s/name' % str(dom2))
  244. if dom_name is not None:
  245. notify_error_qubes_manager(str(dom_name), no_progress_msg)
  246. else:
  247. self.log.warning('dom {!r} still hold more'
  248. ' memory than have assigned ({} > {})'
  249. .format(dom2,
  250. self.domdict[dom2].memory_actual,
  251. mem2))
  252. self.domdict[dom2].slow_memset_react = True
  253. dom_name = self.xs.read('', '/local/domain/%s/name' % str(dom2))
  254. if dom_name is not None:
  255. notify_error_qubes_manager(str(dom_name), slow_memset_react_msg)
  256. self.mem_set(dom, self.get_free_xen_memory() + self.domdict[dom].memory_actual - self.XEN_FREE_MEM_LEFT)
  257. return
  258. self.mem_set(dom, mem)
  259. # for i in self.domdict.keys():
  260. # print 'domain ', i, ' meminfo=', self.domdict[i].meminfo, 'actual mem', self.domdict[i].memory_actual
  261. # print 'domain ', i, 'actual mem', self.domdict[i].memory_actual
  262. # print 'xen free mem', self.get_free_xen_memory()