__init__.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365
  1. # pylint: skip-file
  2. #
  3. # The Qubes OS Project, http://www.qubes-os.org
  4. #
  5. # Copyright (C) 2010 Rafal Wojtczuk <rafal@invisiblethingslab.com>
  6. # Copyright (C) 2013 Marek Marczykowski <marmarek@invisiblethingslab.com>
  7. #
  8. # This library is free software; you can redistribute it and/or
  9. # modify it under the terms of the GNU Lesser General Public
  10. # License as published by the Free Software Foundation; either
  11. # version 2.1 of the License, or (at your option) any later version.
  12. #
  13. # This library is distributed in the hope that it will be useful,
  14. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16. # Lesser General Public License for more details.
  17. #
  18. # You should have received a copy of the GNU Lesser General Public
  19. # License along with this library; if not, see <https://www.gnu.org/licenses/>.
  20. #
  21. import logging
  22. import os
  23. import string
  24. import time
  25. import functools
  26. import xen.lowlevel.xc
  27. import xen.lowlevel.xs
  28. import qubes.qmemman.algo
  29. no_progress_msg="VM refused to give back requested memory"
  30. slow_memset_react_msg="VM didn't give back all requested memory"
  31. class DomainState:
  32. def __init__(self, id):
  33. self.memory_current = 0 # the current memory size
  34. self.memory_actual = None # the current memory allocation (what VM
  35. # is using or can use at any time)
  36. self.memory_maximum = None # the maximum memory size
  37. self.mem_used = None # used memory, computed based on meminfo
  38. self.id = id # domain id
  39. self.last_target = 0 # the last memset target
  40. self.no_progress = False # no react to memset
  41. self.slow_memset_react = False # slow react to memset (after few
  42. # tries still above target)
  43. def __repr__(self):
  44. return self.__dict__.__repr__()
  45. class SystemState(object):
  46. def __init__(self):
  47. self.log = logging.getLogger('qmemman.systemstate')
  48. self.log.debug('SystemState()')
  49. self.domdict = {}
  50. self.xc = xen.lowlevel.xc.xc()
  51. self.xs = xen.lowlevel.xs.xs()
  52. self.BALOON_DELAY = 0.1
  53. self.XEN_FREE_MEM_LEFT = 50*1024*1024
  54. self.XEN_FREE_MEM_MIN = 25*1024*1024
  55. # Overhead of per-page Xen structures, taken from OpenStack
  56. # nova/virt/xenapi/driver.py
  57. # see https://wiki.openstack.org/wiki/XenServer/Overhead
  58. # we divide total and free physical memory by this to get
  59. # "assignable" memory
  60. self.MEM_OVERHEAD_FACTOR = 1.0 / 1.00781
  61. try:
  62. self.ALL_PHYS_MEM = int(self.xc.physinfo()['total_memory']*1024 * self.MEM_OVERHEAD_FACTOR)
  63. except xen.lowlevel.xc.Error:
  64. self.ALL_PHYS_MEM = 0
  65. def add_domain(self, id):
  66. self.log.debug('add_domain(id={!r})'.format(id))
  67. self.domdict[id] = DomainState(id)
  68. # TODO: move to DomainState.__init__
  69. target_str = self.xs.read('', '/local/domain/' + id + '/memory/target')
  70. if target_str:
  71. self.domdict[id].last_target = int(target_str) * 1024
  72. def del_domain(self, id):
  73. self.log.debug('del_domain(id={!r})'.format(id))
  74. self.domdict.pop(id)
  75. def get_free_xen_memory(self):
  76. xen_free = int(self.xc.physinfo()['free_memory']*1024 *
  77. self.MEM_OVERHEAD_FACTOR)
  78. # now check for domains which have assigned more memory than really
  79. # used - do not count it as "free", because domain is free to use it
  80. # at any time
  81. # assumption: self.refresh_memactual was called before
  82. # (so domdict[id].memory_actual is up to date)
  83. assigned_but_unused = functools.reduce(
  84. lambda acc, dom: acc + max(0, dom.last_target-dom.memory_current),
  85. self.domdict.values(),
  86. 0
  87. )
  88. # If, at any time, Xen have less memory than XEN_FREE_MEM_MIN,
  89. # it is a failure of qmemman. Collect as much data as possible to
  90. # debug it
  91. if xen_free < self.XEN_FREE_MEM_MIN:
  92. self.log.error("Xen free = {!r} below acceptable value! "
  93. "assigned_but_unused={!r}, domdict={!r}".format(
  94. xen_free, assigned_but_unused, self.domdict))
  95. elif xen_free < assigned_but_unused+self.XEN_FREE_MEM_MIN:
  96. self.log.error("Xen free = {!r} too small for satisfy assignments! "
  97. "assigned_but_unused={!r}, domdict={!r}".format(
  98. xen_free, assigned_but_unused, self.domdict))
  99. return xen_free - assigned_but_unused
  100. # refresh information on memory assigned to all domains
  101. def refresh_memactual(self):
  102. for domain in self.xc.domain_getinfo():
  103. id = str(domain['domid'])
  104. if id in self.domdict:
  105. # real memory usage
  106. self.domdict[id].memory_current = domain['mem_kb']*1024
  107. # what VM is using or can use
  108. self.domdict[id].memory_actual = max(
  109. self.domdict[id].memory_current,
  110. self.domdict[id].last_target
  111. )
  112. self.domdict[id].memory_maximum = self.xs.read('', '/local/domain/%s/memory/static-max' % str(id))
  113. if self.domdict[id].memory_maximum:
  114. self.domdict[id].memory_maximum = int(self.domdict[id].memory_maximum)*1024
  115. else:
  116. self.domdict[id].memory_maximum = self.ALL_PHYS_MEM
  117. # the previous line used to be
  118. # self.domdict[id].memory_maximum = domain[
  119. # 'maxmem_kb']*1024
  120. # but domain['maxmem_kb'] changes in self.mem_set as well,
  121. # and this results in the memory never increasing
  122. # in fact, the only possible case of nonexisting
  123. # memory/static-max is dom0
  124. # see #307
  125. def clear_outdated_error_markers(self):
  126. # Clear outdated errors
  127. for i in self.domdict.keys():
  128. if self.domdict[i].mem_used is None:
  129. continue
  130. # clear markers excluding VM from memory balance, if:
  131. # - VM have responded to previous request (with some safety margin)
  132. # - VM request more memory than it has assigned
  133. # The second condition avoids starving a VM, even when there is
  134. # some free memory available
  135. if self.domdict[i].memory_actual <= \
  136. self.domdict[i].last_target + self.XEN_FREE_MEM_LEFT/2 or \
  137. self.domdict[i].memory_actual < \
  138. qubes.qmemman.algo.prefmem(self.domdict[i]):
  139. self.domdict[i].slow_memset_react = False
  140. self.domdict[i].no_progress = False
  141. # the below works (and is fast), but then 'xm list' shows unchanged
  142. # memory value
  143. def mem_set(self, id, val):
  144. self.log.info('mem-set domain {} to {}'.format(id, val))
  145. self.domdict[id].last_target = val
  146. # can happen in the middle of domain shutdown
  147. # apparently xc.lowlevel throws exceptions too
  148. try:
  149. self.xc.domain_setmaxmem(int(id), int(val/1024) + 1024) # LIBXL_MAXMEM_CONSTANT=1024
  150. self.xc.domain_set_target_mem(int(id), int(val / 1024))
  151. except:
  152. pass
  153. # VM sees about 16MB memory less, so adjust for it here - qmemman
  154. # handle Xen view of memory
  155. self.xs.write('', '/local/domain/' + id + '/memory/target',
  156. str(int(val/1024 - 16 * 1024)))
  157. # this is called at the end of ballooning, when we have Xen free mem already
  158. # make sure that past mem_set will not decrease Xen free mem
  159. def inhibit_balloon_up(self):
  160. self.log.debug('inhibit_balloon_up()')
  161. for i in self.domdict.keys():
  162. dom = self.domdict[i]
  163. if dom.memory_actual is not None and dom.memory_actual + 200*1024 < dom.last_target:
  164. self.log.info(
  165. 'Preventing balloon up to {}'.format(dom.last_target))
  166. self.mem_set(i, dom.memory_actual)
  167. # perform memory ballooning, across all domains, to add "memsize" to Xen
  168. # free memory
  169. def do_balloon(self, memsize):
  170. self.log.info('do_balloon(memsize={!r})'.format(memsize))
  171. CHECK_PERIOD_S = 3
  172. CHECK_MB_S = 100
  173. niter = 0
  174. prev_memory_actual = None
  175. for i in self.domdict.keys():
  176. self.domdict[i].no_progress = False
  177. #: number of loop iterations for CHECK_PERIOD_S seconds
  178. check_period = max(1, int((CHECK_PERIOD_S + 0.0) / self.BALOON_DELAY))
  179. #: number of free memory bytes expected to get during CHECK_PERIOD_S
  180. #: seconds
  181. check_delta = CHECK_PERIOD_S * CHECK_MB_S * 1024 * 1024
  182. #: helper array for holding free memory size, CHECK_PERIOD_S seconds
  183. #: ago, at every loop iteration
  184. xenfree_ring = [0] * check_period
  185. while True:
  186. self.log.debug('niter={:2d}'.format(niter))
  187. self.refresh_memactual()
  188. xenfree = self.get_free_xen_memory()
  189. self.log.info('xenfree={!r}'.format(xenfree))
  190. if xenfree >= memsize + self.XEN_FREE_MEM_MIN:
  191. self.inhibit_balloon_up()
  192. return True
  193. # fail the request if over past CHECK_PERIOD_S seconds,
  194. # we got less than CHECK_MB_S MB/s on average
  195. ring_slot = niter % check_period
  196. if niter >= check_period and xenfree < xenfree_ring[ring_slot] + check_delta:
  197. return False
  198. xenfree_ring[ring_slot] = xenfree
  199. if prev_memory_actual is not None:
  200. for i in prev_memory_actual.keys():
  201. if prev_memory_actual[i] == self.domdict[i].memory_actual:
  202. # domain not responding to memset requests, remove it
  203. # from donors
  204. self.domdict[i].no_progress = True
  205. self.log.info('domain {} stuck at {}'.format(i, self.domdict[i].memory_actual))
  206. memset_reqs = qubes.qmemman.algo.balloon(memsize + self.XEN_FREE_MEM_LEFT - xenfree, self.domdict)
  207. self.log.info('memset_reqs={!r}'.format(memset_reqs))
  208. if len(memset_reqs) == 0:
  209. return False
  210. prev_memory_actual = {}
  211. for i in memset_reqs:
  212. dom, mem = i
  213. self.mem_set(dom, mem)
  214. prev_memory_actual[dom] = self.domdict[dom].memory_actual
  215. self.log.debug('sleeping for {} s'.format(self.BALOON_DELAY))
  216. time.sleep(self.BALOON_DELAY)
  217. niter = niter + 1
  218. def refresh_meminfo(self, domid, untrusted_meminfo_key):
  219. self.log.debug(
  220. 'refresh_meminfo(domid={}, untrusted_meminfo_key={!r})'.format(
  221. domid, untrusted_meminfo_key))
  222. qubes.qmemman.algo.refresh_meminfo_for_domain(
  223. self.domdict[domid], untrusted_meminfo_key)
  224. self.do_balance()
  225. # is the computed balance request big enough ?
  226. # so that we do not trash with small adjustments
  227. def is_balance_req_significant(self, memset_reqs, xenfree):
  228. self.log.debug(
  229. 'is_balance_req_significant(memset_reqs={}, xenfree={})'.format(
  230. memset_reqs, xenfree))
  231. total_memory_transfer = 0
  232. MIN_TOTAL_MEMORY_TRANSFER = 150*1024*1024
  233. MIN_MEM_CHANGE_WHEN_UNDER_PREF = 15*1024*1024
  234. # If xenfree to low, return immediately
  235. if self.XEN_FREE_MEM_LEFT - xenfree > MIN_MEM_CHANGE_WHEN_UNDER_PREF:
  236. self.log.debug('xenfree is too low, returning')
  237. return True
  238. for rq in memset_reqs:
  239. dom, mem = rq
  240. last_target = self.domdict[dom].last_target
  241. memory_change = mem - last_target
  242. total_memory_transfer += abs(memory_change)
  243. pref = qubes.qmemman.algo.prefmem(self.domdict[dom])
  244. if 0 < last_target < pref and \
  245. memory_change > MIN_MEM_CHANGE_WHEN_UNDER_PREF:
  246. self.log.info(
  247. 'dom {} is below pref, allowing balance'.format(dom))
  248. return True
  249. ret = total_memory_transfer + abs(xenfree - self.XEN_FREE_MEM_LEFT) > MIN_TOTAL_MEMORY_TRANSFER
  250. self.log.debug('is_balance_req_significant return {}'.format(ret))
  251. return ret
  252. def print_stats(self, xenfree, memset_reqs):
  253. for i in self.domdict.keys():
  254. if self.domdict[i].mem_used is not None:
  255. self.log.info('stat: dom {!r} act={} pref={} last_target={}'
  256. '{}{}'.format(i,
  257. self.domdict[i].memory_actual,
  258. qubes.qmemman.algo.prefmem(self.domdict[i]),
  259. self.domdict[i].last_target,
  260. ' no_progress' if self.domdict[i].no_progress else '',
  261. ' slow_memset_react'
  262. if self.domdict[i].slow_memset_react else ''))
  263. self.log.info('stat: xenfree={} memset_reqs={}'.format(xenfree, memset_reqs))
  264. def do_balance(self):
  265. self.log.debug('do_balance()')
  266. if os.path.isfile('/var/run/qubes/do-not-membalance'):
  267. self.log.debug('do-not-membalance file preset, returning')
  268. return
  269. self.refresh_memactual()
  270. self.clear_outdated_error_markers()
  271. xenfree = self.get_free_xen_memory()
  272. memset_reqs = qubes.qmemman.algo.balance(xenfree - self.XEN_FREE_MEM_LEFT, self.domdict)
  273. if not self.is_balance_req_significant(memset_reqs, xenfree):
  274. return
  275. self.print_stats(xenfree, memset_reqs)
  276. prev_memactual = {}
  277. for i in self.domdict.keys():
  278. prev_memactual[i] = self.domdict[i].memory_actual
  279. for rq in memset_reqs:
  280. dom, mem = rq
  281. # Force to always have at least 0.9*self.XEN_FREE_MEM_LEFT (some
  282. # margin for rounding errors). Before giving memory to
  283. # domain, ensure that others have gave it back.
  284. # If not - wait a little.
  285. ntries = 5
  286. while self.get_free_xen_memory() - (mem - self.domdict[dom].memory_actual) < 0.9*self.XEN_FREE_MEM_LEFT:
  287. self.log.debug('do_balance dom={!r} sleeping ntries={}'.format(
  288. dom, ntries))
  289. time.sleep(self.BALOON_DELAY)
  290. self.refresh_memactual()
  291. ntries -= 1
  292. if ntries <= 0:
  293. # Waiting haven't helped; Find which domain get stuck and
  294. # abort balance (after distributing what we have)
  295. for rq2 in memset_reqs:
  296. dom2, mem2 = rq2
  297. if dom2 == dom:
  298. # All donors have been processed
  299. break
  300. # allow some small margin
  301. if self.domdict[dom2].memory_actual > self.domdict[dom2].last_target + self.XEN_FREE_MEM_LEFT/4:
  302. # VM didn't react to memory request at all,
  303. # remove from donors
  304. if prev_memactual[dom2] == self.domdict[dom2].memory_actual:
  305. self.log.warning(
  306. 'dom {!r} didnt react to memory request'
  307. ' (holds {}, requested balloon down to {})'
  308. .format(dom2,
  309. self.domdict[dom2].memory_actual,
  310. mem2))
  311. self.domdict[dom2].no_progress = True
  312. else:
  313. self.log.warning('dom {!r} still hold more'
  314. ' memory than have assigned ({} > {})'
  315. .format(dom2,
  316. self.domdict[dom2].memory_actual,
  317. mem2))
  318. self.domdict[dom2].slow_memset_react = True
  319. self.mem_set(dom, self.get_free_xen_memory() + self.domdict[dom].memory_actual - self.XEN_FREE_MEM_LEFT)
  320. return
  321. self.mem_set(dom, mem)
  322. # for i in self.domdict.keys():
  323. # print 'domain ', i, ' meminfo=', self.domdict[i].mem_used, 'actual mem', self.domdict[i].memory_actual
  324. # print 'domain ', i, 'actual mem', self.domdict[i].memory_actual
  325. # print 'xen free mem', self.get_free_xen_memory()