From c538d536c8b52c04aee2d63fc3dd4622e44da7d5 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Thu, 1 Oct 2015 22:14:35 +0200 Subject: [PATCH] core3: move qmemman This is part of fixing qvm-start. qmemman was moved with minimal touching, mainly module names. Moved function parsing human-readable sizes from core2. This function is wrong, because it treats k/M/G as 1024-based, but leave it for now. --- Makefile | 7 +- core/qubesutils.py | 17 -- {qmemman => etc}/qmemman.conf | 0 linux/systemd/Makefile | 1 + .../systemd}/qubes-qmemman.service | 2 +- qmemman/Makefile | 24 --- qmemman/server.py | 23 --- {qmemman => qubes/qmemman}/.gitignore | 0 .../qmemman.py => qubes/qmemman/__init__.py | 26 ++- .../qmemman_algo.py => qubes/qmemman/algo.py | 2 +- .../qmemman/client.py | 0 .../tools/qmemmand.py | 169 ++++++++++-------- qubes/utils.py | 19 ++ rpm_spec/core-dom0.spec | 12 +- 14 files changed, 142 insertions(+), 160 deletions(-) rename {qmemman => etc}/qmemman.conf (100%) rename {qmemman => linux/systemd}/qubes-qmemman.service (78%) delete mode 100644 qmemman/Makefile delete mode 100755 qmemman/server.py rename {qmemman => qubes/qmemman}/.gitignore (100%) rename qmemman/qmemman.py => qubes/qmemman/__init__.py (93%) mode change 100755 => 100644 rename qmemman/qmemman_algo.py => qubes/qmemman/algo.py (99%) mode change 100755 => 100644 rename qmemman/qmemman_client.py => qubes/qmemman/client.py (100%) mode change 100755 => 100644 rename qmemman/qmemman_server.py => qubes/tools/qmemmand.py (63%) mode change 100755 => 100644 diff --git a/Makefile b/Makefile index ac389f2a..a981f502 100644 --- a/Makefile +++ b/Makefile @@ -41,14 +41,12 @@ rpms-dom0: clean: make -C dispvm clean - make -C qmemman clean all: python setup.py build # make all -C tests # Currently supported only on xen ifeq ($(BACKEND_VMM),xen) - make all -C qmemman make all -C dispvm endif @@ -63,7 +61,8 @@ endif $(MAKE) install -C relaxng ifeq ($(BACKEND_VMM),xen) # Currently supported only on xen - $(MAKE) install -C qmemman + mkdir -p $(DESTDIR)/etc/qubes + cp etc/qmemman.conf $(DESTDIR)/etc/qubes/ endif $(MAKE) install -C dispvm mkdir -p $(DESTDIR)/etc/qubes-rpc/policy @@ -78,9 +77,11 @@ endif cp qubes-rpc/qubes.NotifyTools $(DESTDIR)/etc/qubes-rpc/ cp qubes-rpc/qubes-notify-updates $(DESTDIR)/usr/libexec/qubes/ cp qubes-rpc/qubes-notify-tools $(DESTDIR)/usr/libexec/qubes/ + mkdir -p "$(DESTDIR)$(FILESDIR)" cp vm-config/$(BACKEND_VMM)-vm-template.xml "$(DESTDIR)$(FILESDIR)/vm-template.xml" cp vm-config/$(BACKEND_VMM)-vm-template-hvm.xml "$(DESTDIR)$(FILESDIR)/vm-template-hvm.xml" + mkdir -p $(DESTDIR)$(DATADIR) mkdir -p $(DESTDIR)$(DATADIR)/vm-templates mkdir -p $(DESTDIR)$(DATADIR)/appvms diff --git a/core/qubesutils.py b/core/qubesutils.py index 130faf8a..f17307ec 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -79,23 +79,6 @@ def size_to_human (size): else: return str(round(size/(1024.0*1024*1024),1)) + ' GiB' -def parse_size(size): - units = [ ('K', 1024), ('KB', 1024), - ('M', 1024*1024), ('MB', 1024*1024), - ('G', 1024*1024*1024), ('GB', 1024*1024*1024), - ] - - size = size.strip().upper() - if size.isdigit(): - return int(size) - - for unit, multiplier in units: - if size.endswith(unit): - size = size[:-len(unit)].strip() - return int(size)*multiplier - - raise QubesException("Invalid size: {0}.".format(size)) - def get_disk_usage_one(st): try: return st.st_blocks * BLKSIZE diff --git a/qmemman/qmemman.conf b/etc/qmemman.conf similarity index 100% rename from qmemman/qmemman.conf rename to etc/qmemman.conf diff --git a/linux/systemd/Makefile b/linux/systemd/Makefile index 8f035146..690fb8ab 100644 --- a/linux/systemd/Makefile +++ b/linux/systemd/Makefile @@ -11,3 +11,4 @@ install: cp qubes-vm@.service $(DESTDIR)$(UNITDIR) cp qubes-reload-firewall@.service $(DESTDIR)$(UNITDIR) cp qubes-reload-firewall@.timer $(DESTDIR)$(UNITDIR) + cp qubes-qmemman.service $(DESTDIR)$(UNITDIR) diff --git a/qmemman/qubes-qmemman.service b/linux/systemd/qubes-qmemman.service similarity index 78% rename from qmemman/qubes-qmemman.service rename to linux/systemd/qubes-qmemman.service index 1b8ae566..e8832d03 100644 --- a/qmemman/qubes-qmemman.service +++ b/linux/systemd/qubes-qmemman.service @@ -4,7 +4,7 @@ After=qubes-core.service [Service] Type=notify -ExecStart=/usr/lib/qubes/qmemman_daemon.py +ExecStart=/usr/bin/qmemmand StandardOutput=syslog [Install] diff --git a/qmemman/Makefile b/qmemman/Makefile deleted file mode 100644 index 6c04a458..00000000 --- a/qmemman/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -PYTHON_QUBESPATH = $(PYTHON_SITEPATH)/qubes -SYSCONFDIR ?= /etc -UNITDIR ?= /usr/lib/systemd/system -all: - python -m compileall . - python -O -m compileall . - -clean: - rm -f *.pyo - -install: -ifndef PYTHON_SITEPATH - $(error PYTHON_SITEPATH not defined) -endif - mkdir -p $(DESTDIR)$(PYTHON_QUBESPATH) - cp qmemman*py $(DESTDIR)$(PYTHON_QUBESPATH) - cp qmemman*py[co] $(DESTDIR)$(PYTHON_QUBESPATH) - mkdir -p $(DESTDIR)$(SYSCONFDIR)/qubes - cp qmemman.conf $(DESTDIR)$(SYSCONFDIR)/qubes/ - mkdir -p $(DESTDIR)/usr/lib/qubes - cp server.py $(DESTDIR)/usr/lib/qubes/qmemman_daemon.py - mkdir -p $(DESTDIR)$(UNITDIR) - cp qubes-qmemman.service $(DESTDIR)$(UNITDIR) - diff --git a/qmemman/server.py b/qmemman/server.py deleted file mode 100755 index a5447425..00000000 --- a/qmemman/server.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python2 -# -*- coding: utf-8 -*- -# -# The Qubes OS Project, http://www.qubes-os.org -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# -from qubes.qmemman_server import QMemmanServer - -QMemmanServer.main() diff --git a/qmemman/.gitignore b/qubes/qmemman/.gitignore similarity index 100% rename from qmemman/.gitignore rename to qubes/qmemman/.gitignore diff --git a/qmemman/qmemman.py b/qubes/qmemman/__init__.py old mode 100755 new mode 100644 similarity index 93% rename from qmemman/qmemman.py rename to qubes/qmemman/__init__.py index f7d7efa4..f0e9109f --- a/qmemman/qmemman.py +++ b/qubes/qmemman/__init__.py @@ -31,7 +31,7 @@ import xen.lowlevel.xc import xen.lowlevel.xs import qubes -import qubes.qmemman_algo +import qubes.qmemman.algo no_progress_msg="VM refused to give back requested memory" @@ -102,14 +102,24 @@ class SystemState(object): self.domdict[i].memory_actual <= self.domdict[i].last_target + self.XEN_FREE_MEM_LEFT/4: dom_name = self.xs.read('', '/local/domain/%s/name' % str(i)) if dom_name is not None: - clear_error_qubes_manager(dom_name, slow_memset_react_msg) + try: + qubes.Qubes()[str(dom_name)].fire_event( + 'status:no-error', 'no-error', + slow_memset_react_msg) + except LookupError: + pass self.domdict[i].slow_memset_react = False if self.domdict[i].no_progress and \ self.domdict[i].memory_actual <= self.domdict[i].last_target + self.XEN_FREE_MEM_LEFT/4: dom_name = self.xs.read('', '/local/domain/%s/name' % str(i)) if dom_name is not None: - clear_error_qubes_manager(dom_name, no_progress_msg) + try: + qubes.Qubes()[str(dom_name)].fire_event( + 'status:no-error', 'no-error', + no_progress_msg) + except LookupError: + pass self.domdict[i].no_progress = False #the below works (and is fast), but then 'xm list' shows unchanged memory value @@ -160,7 +170,7 @@ class SystemState(object): #domain not responding to memset requests, remove it from donors self.domdict[i].no_progress = True self.log.info('domain {} stuck at {}'.format(i, self.domdict[i].memory_actual)) - memset_reqs = qmemman_algo.balloon(memsize + self.XEN_FREE_MEM_LEFT - xenfree, self.domdict) + memset_reqs = qubes.qmemman.algo.balloon(memsize + self.XEN_FREE_MEM_LEFT - xenfree, self.domdict) self.log.info('memset_reqs={!r}'.format(memset_reqs)) if niter > MAX_TRIES or len(memset_reqs) == 0: return False @@ -178,7 +188,7 @@ class SystemState(object): 'refresh_meminfo(domid={}, untrusted_meminfo_key={!r})'.format( domid, untrusted_meminfo_key)) - qmemman_algo.refresh_meminfo_for_domain( + qubes.qmemman.algo.refresh_meminfo_for_domain( self.domdict[domid], untrusted_meminfo_key) self.do_balance() @@ -203,7 +213,7 @@ class SystemState(object): last_target = self.domdict[dom].last_target memory_change = mem - last_target total_memory_transfer += abs(memory_change) - pref = qmemman_algo.prefmem(self.domdict[dom]) + pref = qubes.qmemman.algo.prefmem(self.domdict[dom]) if last_target > 0 and last_target < pref and memory_change > MIN_MEM_CHANGE_WHEN_UNDER_PREF: self.log.info( @@ -220,7 +230,7 @@ class SystemState(object): if self.domdict[i].meminfo is not None: self.log.info('stat: dom {!r} act={} pref={}'.format(i, self.domdict[i].memory_actual, - qmemman_algo.prefmem(self.domdict[i]))) + qubes.qmemman.algo.prefmem(self.domdict[i]))) self.log.info('stat: xenfree={} memset_reqs={}'.format(xenfree, memset_reqs)) @@ -234,7 +244,7 @@ class SystemState(object): self.refresh_memactual() self.clear_outdated_error_markers() xenfree = self.get_free_xen_memory() - memset_reqs = qmemman_algo.balance(xenfree - self.XEN_FREE_MEM_LEFT, self.domdict) + memset_reqs = qubes.qmemman.algo.balance(xenfree - self.XEN_FREE_MEM_LEFT, self.domdict) if not self.is_balance_req_significant(memset_reqs, xenfree): return diff --git a/qmemman/qmemman_algo.py b/qubes/qmemman/algo.py old mode 100755 new mode 100644 similarity index 99% rename from qmemman/qmemman_algo.py rename to qubes/qmemman/algo.py index 271ee1fa..9c77b071 --- a/qmemman/qmemman_algo.py +++ b/qubes/qmemman/algo.py @@ -126,7 +126,7 @@ def balloon(memsize, domain_dictionary): log.info('balloon: dom {} has actual memory {}'.format(i, domain_dictionary[i].memory_actual)) donors.append((i,-need)) - available-=need + available-=need log.info('req={} avail={} donors={!r}'.format(memsize, available, donors)) diff --git a/qmemman/qmemman_client.py b/qubes/qmemman/client.py old mode 100755 new mode 100644 similarity index 100% rename from qmemman/qmemman_client.py rename to qubes/qmemman/client.py diff --git a/qmemman/qmemman_server.py b/qubes/tools/qmemmand.py old mode 100755 new mode 100644 similarity index 63% rename from qmemman/qmemman_server.py rename to qubes/tools/qmemmand.py index cf1bb2fb..b96ab94f --- a/qmemman/qmemman_server.py +++ b/qubes/tools/qmemmand.py @@ -20,31 +20,29 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # +import ConfigParser import SocketServer -import thread -import time -import xen.lowlevel.xs -import sys -import os -import socket -from qmemman import SystemState -import qmemman_algo -from ConfigParser import SafeConfigParser -from optparse import OptionParser -from qubesutils import parse_size - import logging import logging.handlers +import os +import socket +import sys +import thread -config_path = '/etc/qubes/qmemman.conf' -SOCK_PATH='/var/run/qubes/qmemman.sock' -LOG_PATH='/var/log/qubes/qmemman.log' +import xen.lowlevel.xs -system_state = SystemState() +import qubes.qmemman +import qubes.qmemman.algo +import qubes.utils + +SOCK_PATH = '/var/run/qubes/qmemman.sock' +LOG_PATH = '/var/log/qubes/qmemman.log' + +system_state = qubes.qmemman.SystemState() global_lock = thread.allocate_lock() def only_in_first_list(l1, l2): - ret=[] + ret = [] for i in l1: if not i in l2: ret.append(i) @@ -53,13 +51,13 @@ def only_in_first_list(l1, l2): def get_domain_meminfo_key(domain_id): return '/local/domain/'+domain_id+'/memory/meminfo' - -class WatchType: + +class WatchType(object): def __init__(self, fn, param): self.fn = fn self.param = param -class XS_Watcher: +class XS_Watcher(object): def __init__(self): self.log = logging.getLogger('qmemman.daemon.xswatcher') self.log.debug('XS_Watcher()') @@ -104,7 +102,8 @@ class XS_Watcher: def meminfo_changed(self, domain_id): self.log.debug('meminfo_changed(domain_id={!r})'.format(domain_id)) - untrusted_meminfo_key = self.handle.read('', get_domain_meminfo_key(domain_id)) + untrusted_meminfo_key = self.handle.read( + '', get_domain_meminfo_key(domain_id)) if untrusted_meminfo_key == None or untrusted_meminfo_key == '': return @@ -171,72 +170,88 @@ class QMemmanReqHandler(SocketServer.BaseRequestHandler): # XXX no release of lock? -def start_server(server): - server.serve_forever() +parser = qubes.tools.get_parser_base() -class QMemmanServer: - @staticmethod - def main(): - # setup logging - ha_syslog = logging.handlers.SysLogHandler('/dev/log') - ha_syslog.setFormatter( - logging.Formatter('%(name)s[%(process)d]: %(message)s')) - logging.root.addHandler(ha_syslog) +parser.add_argument('--config', '-c', metavar='FILE', + action='store', default='/etc/qubes/qmemman.conf', + help='qmemman config file') - # leave log for backwards compatibility - ha_file = logging.FileHandler(LOG_PATH) +parser.add_argument('--foreground', + action='store_true', default=False, + help='do not close stdio') + + +def main(): + args = parser.parse_args() + + # setup logging + ha_syslog = logging.handlers.SysLogHandler('/dev/log') + ha_syslog.setFormatter( + logging.Formatter('%(name)s[%(process)d]: %(message)s')) + logging.root.addHandler(ha_syslog) + + # leave log for backwards compatibility + ha_file = logging.FileHandler(LOG_PATH) + ha_file.setFormatter( + logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(message)s')) + logging.root.addHandler(ha_file) + + if args.foreground: + ha_stderr = logging.StreamHandler(sys.stderr) ha_file.setFormatter( logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(message)s')) - logging.root.addHandler(ha_file) - - log = logging.getLogger('qmemman.daemon') - - usage = "usage: %prog [options]" - parser = OptionParser(usage) - parser.add_option("-c", "--config", action="store", dest="config", default=config_path) - (options, args) = parser.parse_args() - + logging.root.addHandler(ha_stderr) + else: # close io - sys.stdin.close() sys.stdout.close() sys.stderr.close() - config = SafeConfigParser({ - 'vm-min-mem': str(qmemman_algo.MIN_PREFMEM), - 'dom0-mem-boost': str(qmemman_algo.DOM0_MEM_BOOST), - 'cache-margin-factor': str(qmemman_algo.CACHE_FACTOR) - }) - config.read(options.config) - if config.has_section('global'): - qmemman_algo.MIN_PREFMEM = parse_size(config.get('global', 'vm-min-mem')) - qmemman_algo.DOM0_MEM_BOOST = parse_size(config.get('global', 'dom0-mem-boost')) - qmemman_algo.CACHE_FACTOR = config.getfloat('global', 'cache-margin-factor') + sys.stdin.close() - log.info('MIN_PREFMEM={qmemman_algo.MIN_PREFMEM}' - ' DOM0_MEM_BOOST={qmemman_algo.DOM0_MEM_BOOST}' - ' CACHE_FACTOR={qmemman_algo.CACHE_FACTOR}'.format( - qmemman_algo=qmemman_algo)) + logging.root.setLevel((args.quiet - args.verbose) * 10 + logging.WARNING) - try: - os.unlink(SOCK_PATH) - except: - pass + log = logging.getLogger('qmemman.daemon') - log.debug('instantiating server') - os.umask(0) - server = SocketServer.UnixStreamServer(SOCK_PATH, QMemmanReqHandler) - os.umask(077) + config = ConfigParser.SafeConfigParser({ + 'vm-min-mem': str(qubes.qmemman.algo.MIN_PREFMEM), + 'dom0-mem-boost': str(qubes.qmemman.algo.DOM0_MEM_BOOST), + 'cache-margin-factor': str(qubes.qmemman.algo.CACHE_FACTOR) + }) + config.read(args.config) - # notify systemd - nofity_socket = os.getenv('NOTIFY_SOCKET') - if nofity_socket: - log.debug('notifying systemd') - s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - if nofity_socket.startswith('@'): - nofity_socket = '\0%s' % nofity_socket[1:] - s.connect(nofity_socket) - s.sendall("READY=1") - s.close() + if config.has_section('global'): + qubes.qmemman.algo.MIN_PREFMEM = \ + qubes.utils.parse_size(config.get('global', 'vm-min-mem')) + qubes.qmemman.algo.DOM0_MEM_BOOST = \ + qubes.utils.parse_size(config.get('global', 'dom0-mem-boost')) + qubes.qmemman.algo.CACHE_FACTOR = \ + config.getfloat('global', 'cache-margin-factor') - thread.start_new_thread(start_server, tuple([server])) - XS_Watcher().watch_loop() + log.info('MIN_PREFMEM={algo.MIN_PREFMEM}' + ' DOM0_MEM_BOOST={algo.DOM0_MEM_BOOST}' + ' CACHE_FACTOR={algo.CACHE_FACTOR}'.format( + algo=qubes.qmemman.algo)) + + try: + os.unlink(SOCK_PATH) + except: + pass + + log.debug('instantiating server') + os.umask(0) + server = SocketServer.UnixStreamServer(SOCK_PATH, QMemmanReqHandler) + os.umask(077) + + # notify systemd + nofity_socket = os.getenv('NOTIFY_SOCKET') + if nofity_socket: + log.debug('notifying systemd') + s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + if nofity_socket.startswith('@'): + nofity_socket = '\0%s' % nofity_socket[1:] + s.connect(nofity_socket) + s.sendall("READY=1") + s.close() + + thread.start_new_thread(server.serve_forever, ()) + XS_Watcher().watch_loop() diff --git a/qubes/utils.py b/qubes/utils.py index ae38f985..86cbd0a0 100644 --- a/qubes/utils.py +++ b/qubes/utils.py @@ -82,3 +82,22 @@ def format_doc(docstring): settings=None, settings_spec=None, settings_overrides=None, config_section=None, enable_exit_status=None) return pub.writer.document.astext() + +# FIXME those are wrong, k/M/G are SI prefixes and means 10**3 +# maybe adapt https://code.activestate.com/recipes/578019 +def parse_size(size): + units = [ ('K', 1024), ('KB', 1024), + ('M', 1024*1024), ('MB', 1024*1024), + ('G', 1024*1024*1024), ('GB', 1024*1024*1024), + ] + + size = size.strip().upper() + if size.isdigit(): + return int(size) + + for unit, multiplier in units: + if size.endswith(unit): + size = size[:-len(unit)].strip() + return int(size)*multiplier + + raise QubesException("Invalid size: {0}.".format(size)) diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index 5ae927ce..1227cb46 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -188,6 +188,7 @@ fi %config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/qmemman.conf /usr/bin/qvm-* /usr/bin/qubes-* +/usr/bin/qmemmand %dir %{python_sitelib}/qubes-*.egg-info %{python_sitelib}/qubes-*.egg-info/* @@ -221,6 +222,7 @@ fi %dir %{python_sitelib}/qubes/tools %{python_sitelib}/qubes/tools/__init__.py* +%{python_sitelib}/qubes/tools/qmemmand.py* %{python_sitelib}/qubes/tools/qubes_create.py* %{python_sitelib}/qubes/tools/qvm_create.py* %{python_sitelib}/qubes/tools/qvm_ls.py* @@ -249,15 +251,13 @@ fi %{python_sitelib}/qubes/tests/tools/init.py* %{python_sitelib}/qubes/tests/tools/qvm_ls.py* -# qmemman -%{python_sitelib}/qubes/qmemman.py* -%{python_sitelib}/qubes/qmemman_algo.py* -%{python_sitelib}/qubes/qmemman_client.py* -%{python_sitelib}/qubes/qmemman_server.py* +%dir %{python_sitelib}/qubes/qmemman +%{python_sitelib}/qubes/qmemman/__init__.py* +%{python_sitelib}/qubes/qmemman/algo.py* +%{python_sitelib}/qubes/qmemman/client.py* /usr/lib/qubes/unbind-pci-device.sh /usr/lib/qubes/cleanup-dispvms -/usr/lib/qubes/qmemman_daemon.py* /usr/lib/qubes/qfile-daemon-dvm* /usr/lib/qubes/block-cleaner-daemon.py* /usr/lib/qubes/vusb-ctl.py*