core3: move qmemman

This is part of fixing qvm-start.

qmemman was moved with minimal touching, mainly module names.

Moved function parsing human-readable sizes from core2. This function is
wrong, because it treats k/M/G as 1024-based, but leave it for now.

Makefile

@@ -41,14 +41,12 @@ rpms-dom0:
 
 clean:
 	make -C dispvm clean
-	make -C qmemman clean
 
 all:
 	python setup.py build
 #	make all -C tests
 	# Currently supported only on xen
 ifeq ($(BACKEND_VMM),xen)
-	make all -C qmemman
 	make all -C dispvm
 endif
 
@@ -63,7 +61,8 @@ endif
 	$(MAKE) install -C relaxng
 ifeq ($(BACKEND_VMM),xen)
 	# Currently supported only on xen
-	$(MAKE) install -C qmemman
+	mkdir -p $(DESTDIR)/etc/qubes
+	cp etc/qmemman.conf $(DESTDIR)/etc/qubes/
 endif
 	$(MAKE) install -C dispvm
 	mkdir -p $(DESTDIR)/etc/qubes-rpc/policy
@@ -78,9 +77,11 @@ endif
 	cp qubes-rpc/qubes.NotifyTools $(DESTDIR)/etc/qubes-rpc/
 	cp qubes-rpc/qubes-notify-updates $(DESTDIR)/usr/libexec/qubes/
 	cp qubes-rpc/qubes-notify-tools $(DESTDIR)/usr/libexec/qubes/
+
 	mkdir -p "$(DESTDIR)$(FILESDIR)"
 	cp vm-config/$(BACKEND_VMM)-vm-template.xml "$(DESTDIR)$(FILESDIR)/vm-template.xml"
 	cp vm-config/$(BACKEND_VMM)-vm-template-hvm.xml "$(DESTDIR)$(FILESDIR)/vm-template-hvm.xml"
+
 	mkdir -p $(DESTDIR)$(DATADIR)
 	mkdir -p $(DESTDIR)$(DATADIR)/vm-templates
 	mkdir -p $(DESTDIR)$(DATADIR)/appvms

core/qubesutils.py

@@ -79,23 +79,6 @@ def size_to_human (size):
     else:
         return str(round(size/(1024.0*1024*1024),1)) + ' GiB'
 
-def parse_size(size):
-    units = [ ('K', 1024), ('KB', 1024),
-        ('M', 1024*1024), ('MB', 1024*1024),
-        ('G', 1024*1024*1024), ('GB', 1024*1024*1024),
-    ]
-
-    size = size.strip().upper()
-    if size.isdigit():
-        return int(size)
-
-    for unit, multiplier in units:
-        if size.endswith(unit):
-            size = size[:-len(unit)].strip()
-            return int(size)*multiplier
-
-    raise QubesException("Invalid size: {0}.".format(size))
-
 def get_disk_usage_one(st):
     try:
         return st.st_blocks * BLKSIZE

linux/systemd/Makefile

@@ -11,3 +11,4 @@ install:
 	cp qubes-vm@.service $(DESTDIR)$(UNITDIR)
 	cp qubes-reload-firewall@.service $(DESTDIR)$(UNITDIR)
 	cp qubes-reload-firewall@.timer $(DESTDIR)$(UNITDIR)
+	cp qubes-qmemman.service $(DESTDIR)$(UNITDIR)

qmemman/qubes-qmemman.service → linux/systemd/qubes-qmemman.service

@@ -4,7 +4,7 @@ After=qubes-core.service
 
 [Service]
 Type=notify
-ExecStart=/usr/lib/qubes/qmemman_daemon.py
+ExecStart=/usr/bin/qmemmand
 StandardOutput=syslog
 
 [Install]

qmemman/Makefile

@@ -1,24 +0,0 @@
-PYTHON_QUBESPATH = $(PYTHON_SITEPATH)/qubes
-SYSCONFDIR ?= /etc
-UNITDIR ?= /usr/lib/systemd/system
-all:
-	python -m compileall .
-	python -O -m compileall .
-
-clean:
-	rm -f *.pyo 
-
-install:
-ifndef PYTHON_SITEPATH
-	$(error PYTHON_SITEPATH not defined)
-endif
-	mkdir -p $(DESTDIR)$(PYTHON_QUBESPATH)
-	cp qmemman*py $(DESTDIR)$(PYTHON_QUBESPATH)
-	cp qmemman*py[co] $(DESTDIR)$(PYTHON_QUBESPATH)
-	mkdir -p $(DESTDIR)$(SYSCONFDIR)/qubes
-	cp qmemman.conf $(DESTDIR)$(SYSCONFDIR)/qubes/
-	mkdir -p $(DESTDIR)/usr/lib/qubes
-	cp server.py $(DESTDIR)/usr/lib/qubes/qmemman_daemon.py
-	mkdir -p $(DESTDIR)$(UNITDIR)
-	cp qubes-qmemman.service $(DESTDIR)$(UNITDIR)
-

qmemman/server.py

@@ -1,23 +0,0 @@
-#!/usr/bin/python2
-# -*- coding: utf-8 -*-
-#
-# The Qubes OS Project, http://www.qubes-os.org
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-#
-#
-from qubes.qmemman_server import QMemmanServer
-
-QMemmanServer.main()

qmemman/qmemman.py → qubes/qmemman/__init__.py

@@ -31,7 +31,7 @@ import xen.lowlevel.xc
 import xen.lowlevel.xs
 
 import qubes
-import qubes.qmemman_algo
+import qubes.qmemman.algo
 
 
 no_progress_msg="VM refused to give back requested memory"
@@ -102,14 +102,24 @@ class SystemState(object):
                     self.domdict[i].memory_actual <= self.domdict[i].last_target + self.XEN_FREE_MEM_LEFT/4:
                 dom_name = self.xs.read('', '/local/domain/%s/name' % str(i))
                 if dom_name is not None:
-                    clear_error_qubes_manager(dom_name, slow_memset_react_msg)
+                    try:
+                        qubes.Qubes()[str(dom_name)].fire_event(
+                            'status:no-error', 'no-error',
+                            slow_memset_react_msg)
+                    except LookupError:
+                        pass
                 self.domdict[i].slow_memset_react = False
 
             if self.domdict[i].no_progress and \
                     self.domdict[i].memory_actual <= self.domdict[i].last_target + self.XEN_FREE_MEM_LEFT/4:
                 dom_name = self.xs.read('', '/local/domain/%s/name' % str(i))
                 if dom_name is not None:
-                    clear_error_qubes_manager(dom_name, no_progress_msg)
+                    try:
+                        qubes.Qubes()[str(dom_name)].fire_event(
+                            'status:no-error', 'no-error',
+                            no_progress_msg)
+                    except LookupError:
+                        pass
                 self.domdict[i].no_progress = False
 
 #the below works (and is fast), but then 'xm list' shows unchanged memory value
@@ -160,7 +170,7 @@ class SystemState(object):
                         #domain not responding to memset requests, remove it from donors
                         self.domdict[i].no_progress = True
                         self.log.info('domain {} stuck at {}'.format(i, self.domdict[i].memory_actual))
-            memset_reqs = qmemman_algo.balloon(memsize + self.XEN_FREE_MEM_LEFT - xenfree, self.domdict)
+            memset_reqs = qubes.qmemman.algo.balloon(memsize + self.XEN_FREE_MEM_LEFT - xenfree, self.domdict)
             self.log.info('memset_reqs={!r}'.format(memset_reqs))
             if niter > MAX_TRIES or len(memset_reqs) == 0:
                 return False
@@ -178,7 +188,7 @@ class SystemState(object):
             'refresh_meminfo(domid={}, untrusted_meminfo_key={!r})'.format(
                 domid, untrusted_meminfo_key))
 
-        qmemman_algo.refresh_meminfo_for_domain(
+        qubes.qmemman.algo.refresh_meminfo_for_domain(
             self.domdict[domid], untrusted_meminfo_key)
         self.do_balance()
 
@@ -203,7 +213,7 @@ class SystemState(object):
             last_target = self.domdict[dom].last_target
             memory_change = mem - last_target
             total_memory_transfer += abs(memory_change)
-            pref = qmemman_algo.prefmem(self.domdict[dom])
+            pref = qubes.qmemman.algo.prefmem(self.domdict[dom])
 
             if last_target > 0 and last_target < pref and memory_change > MIN_MEM_CHANGE_WHEN_UNDER_PREF:
                 self.log.info(
@@ -220,7 +230,7 @@ class SystemState(object):
             if self.domdict[i].meminfo is not None:
                 self.log.info('stat: dom {!r} act={} pref={}'.format(i,
                     self.domdict[i].memory_actual,
-                    qmemman_algo.prefmem(self.domdict[i])))
+                    qubes.qmemman.algo.prefmem(self.domdict[i])))
 
         self.log.info('stat: xenfree={} memset_reqs={}'.format(xenfree, memset_reqs))
 
@@ -234,7 +244,7 @@ class SystemState(object):
         self.refresh_memactual()
         self.clear_outdated_error_markers()
         xenfree = self.get_free_xen_memory()
-        memset_reqs = qmemman_algo.balance(xenfree - self.XEN_FREE_MEM_LEFT, self.domdict)
+        memset_reqs = qubes.qmemman.algo.balance(xenfree - self.XEN_FREE_MEM_LEFT, self.domdict)
         if not self.is_balance_req_significant(memset_reqs, xenfree):
             return
 

qmemman/qmemman_algo.py → qubes/qmemman/algo.py

@@ -126,7 +126,7 @@ def balloon(memsize, domain_dictionary):
             log.info('balloon: dom {} has actual memory {}'.format(i,
                 domain_dictionary[i].memory_actual))
             donors.append((i,-need))
-            available-=need   
+            available-=need
 
     log.info('req={} avail={} donors={!r}'.format(memsize, available, donors))
 

qmemman/qmemman_server.py → qubes/tools/qmemmand.py

@@ -20,31 +20,29 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 #
+import ConfigParser
 import SocketServer
-import thread
-import time
-import xen.lowlevel.xs
-import sys
+import logging
+import logging.handlers
 import os
 import socket
-from qmemman import SystemState
-import qmemman_algo
-from ConfigParser import SafeConfigParser
-from optparse import OptionParser
-from qubesutils import parse_size
+import sys
+import thread
 
-import logging
-import logging.handlers
+import xen.lowlevel.xs
+
+import qubes.qmemman
+import qubes.qmemman.algo
+import qubes.utils
 
-config_path = '/etc/qubes/qmemman.conf'
-SOCK_PATH='/var/run/qubes/qmemman.sock'
-LOG_PATH='/var/log/qubes/qmemman.log'
+SOCK_PATH = '/var/run/qubes/qmemman.sock'
+LOG_PATH = '/var/log/qubes/qmemman.log'
 
-system_state = SystemState()
+system_state = qubes.qmemman.SystemState()
 global_lock = thread.allocate_lock()
 
 def only_in_first_list(l1, l2):
-    ret=[]
+    ret = []
     for i in l1:
         if not i in l2:
             ret.append(i)
@@ -53,13 +51,13 @@ def only_in_first_list(l1, l2):
 def get_domain_meminfo_key(domain_id):
     return '/local/domain/'+domain_id+'/memory/meminfo'
 
-                    
-class WatchType:
+
+class WatchType(object):
     def __init__(self, fn, param):
         self.fn = fn
         self.param = param
 
-class XS_Watcher:
+class XS_Watcher(object):
     def __init__(self):
         self.log = logging.getLogger('qmemman.daemon.xswatcher')
         self.log.debug('XS_Watcher()')
@@ -104,7 +102,8 @@ class XS_Watcher:
 
     def meminfo_changed(self, domain_id):
         self.log.debug('meminfo_changed(domain_id={!r})'.format(domain_id))
-        untrusted_meminfo_key = self.handle.read('', get_domain_meminfo_key(domain_id))
+        untrusted_meminfo_key = self.handle.read(
+            '', get_domain_meminfo_key(domain_id))
         if untrusted_meminfo_key == None or untrusted_meminfo_key == '':
             return
 
@@ -171,72 +170,88 @@ class QMemmanReqHandler(SocketServer.BaseRequestHandler):
             # XXX no release of lock?
 
 
-def start_server(server):
-    server.serve_forever()
+parser = qubes.tools.get_parser_base()
 
-class QMemmanServer:
-    @staticmethod          
-    def main():
-        # setup logging
-        ha_syslog = logging.handlers.SysLogHandler('/dev/log')
-        ha_syslog.setFormatter(
-            logging.Formatter('%(name)s[%(process)d]: %(message)s'))
-        logging.root.addHandler(ha_syslog)
+parser.add_argument('--config', '-c', metavar='FILE',
+    action='store', default='/etc/qubes/qmemman.conf',
+    help='qmemman config file')
 
-        # leave log for backwards compatibility
-        ha_file = logging.FileHandler(LOG_PATH)
-        ha_file.setFormatter(
-            logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(message)s'))
-        logging.root.addHandler(ha_file)
+parser.add_argument('--foreground',
+    action='store_true', default=False,
+    help='do not close stdio')
 
-        log = logging.getLogger('qmemman.daemon')
 
-        usage = "usage: %prog [options]"
-        parser = OptionParser(usage)
-        parser.add_option("-c", "--config", action="store", dest="config", default=config_path)
-        (options, args) = parser.parse_args()
+def main():
+    args = parser.parse_args()
 
+    # setup logging
+    ha_syslog = logging.handlers.SysLogHandler('/dev/log')
+    ha_syslog.setFormatter(
+        logging.Formatter('%(name)s[%(process)d]: %(message)s'))
+    logging.root.addHandler(ha_syslog)
+
+    # leave log for backwards compatibility
+    ha_file = logging.FileHandler(LOG_PATH)
+    ha_file.setFormatter(
+        logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(message)s'))
+    logging.root.addHandler(ha_file)
+
+    if args.foreground:
+        ha_stderr = logging.StreamHandler(sys.stderr)
+        ha_file.setFormatter(
+            logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(message)s'))
+        logging.root.addHandler(ha_stderr)
+    else:
         # close io
-        sys.stdin.close()
         sys.stdout.close()
         sys.stderr.close()
 
-        config = SafeConfigParser({
-                'vm-min-mem': str(qmemman_algo.MIN_PREFMEM),
-                'dom0-mem-boost': str(qmemman_algo.DOM0_MEM_BOOST),
-                'cache-margin-factor': str(qmemman_algo.CACHE_FACTOR)
-                })
-        config.read(options.config)
-        if config.has_section('global'):
-            qmemman_algo.MIN_PREFMEM = parse_size(config.get('global', 'vm-min-mem'))
-            qmemman_algo.DOM0_MEM_BOOST = parse_size(config.get('global', 'dom0-mem-boost'))
-            qmemman_algo.CACHE_FACTOR = config.getfloat('global', 'cache-margin-factor')
-
-        log.info('MIN_PREFMEM={qmemman_algo.MIN_PREFMEM}'
-            ' DOM0_MEM_BOOST={qmemman_algo.DOM0_MEM_BOOST}'
-            ' CACHE_FACTOR={qmemman_algo.CACHE_FACTOR}'.format(
-                qmemman_algo=qmemman_algo))
-
-        try:
-            os.unlink(SOCK_PATH)
-        except:
-            pass
-
-        log.debug('instantiating server')
-        os.umask(0)
-        server = SocketServer.UnixStreamServer(SOCK_PATH, QMemmanReqHandler)
-        os.umask(077)
-
-        # notify systemd
-        nofity_socket = os.getenv('NOTIFY_SOCKET')
-        if nofity_socket:
-            log.debug('notifying systemd')
-            s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
-            if nofity_socket.startswith('@'):
-                nofity_socket = '\0%s' % nofity_socket[1:]
-            s.connect(nofity_socket)
-            s.sendall("READY=1")
-            s.close()
-
-        thread.start_new_thread(start_server, tuple([server]))
-        XS_Watcher().watch_loop()
+    sys.stdin.close()
+
+    logging.root.setLevel((args.quiet - args.verbose) * 10 + logging.WARNING)
+
+    log = logging.getLogger('qmemman.daemon')
+
+    config = ConfigParser.SafeConfigParser({
+            'vm-min-mem': str(qubes.qmemman.algo.MIN_PREFMEM),
+            'dom0-mem-boost': str(qubes.qmemman.algo.DOM0_MEM_BOOST),
+            'cache-margin-factor': str(qubes.qmemman.algo.CACHE_FACTOR)
+            })
+    config.read(args.config)
+
+    if config.has_section('global'):
+        qubes.qmemman.algo.MIN_PREFMEM = \
+            qubes.utils.parse_size(config.get('global', 'vm-min-mem'))
+        qubes.qmemman.algo.DOM0_MEM_BOOST = \
+            qubes.utils.parse_size(config.get('global', 'dom0-mem-boost'))
+        qubes.qmemman.algo.CACHE_FACTOR = \
+            config.getfloat('global', 'cache-margin-factor')
+
+    log.info('MIN_PREFMEM={algo.MIN_PREFMEM}'
+        ' DOM0_MEM_BOOST={algo.DOM0_MEM_BOOST}'
+        ' CACHE_FACTOR={algo.CACHE_FACTOR}'.format(
+            algo=qubes.qmemman.algo))
+
+    try:
+        os.unlink(SOCK_PATH)
+    except:
+        pass
+
+    log.debug('instantiating server')
+    os.umask(0)
+    server = SocketServer.UnixStreamServer(SOCK_PATH, QMemmanReqHandler)
+    os.umask(077)
+
+    # notify systemd
+    nofity_socket = os.getenv('NOTIFY_SOCKET')
+    if nofity_socket:
+        log.debug('notifying systemd')
+        s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+        if nofity_socket.startswith('@'):
+            nofity_socket = '\0%s' % nofity_socket[1:]
+        s.connect(nofity_socket)
+        s.sendall("READY=1")
+        s.close()
+
+    thread.start_new_thread(server.serve_forever, ())
+    XS_Watcher().watch_loop()

qubes/utils.py

@@ -82,3 +82,22 @@ def format_doc(docstring):
         settings=None, settings_spec=None, settings_overrides=None,
         config_section=None, enable_exit_status=None)
     return pub.writer.document.astext()
+
+# FIXME those are wrong, k/M/G are SI prefixes and means 10**3
+# maybe adapt https://code.activestate.com/recipes/578019
+def parse_size(size):
+    units = [ ('K', 1024), ('KB', 1024),
+        ('M', 1024*1024), ('MB', 1024*1024),
+        ('G', 1024*1024*1024), ('GB', 1024*1024*1024),
+    ]
+
+    size = size.strip().upper()
+    if size.isdigit():
+        return int(size)
+
+    for unit, multiplier in units:
+        if size.endswith(unit):
+            size = size[:-len(unit)].strip()
+            return int(size)*multiplier
+
+    raise QubesException("Invalid size: {0}.".format(size))

rpm_spec/core-dom0.spec

@@ -188,6 +188,7 @@ fi
 %config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/qmemman.conf
 /usr/bin/qvm-*
 /usr/bin/qubes-*
+/usr/bin/qmemmand
 
 %dir %{python_sitelib}/qubes-*.egg-info
 %{python_sitelib}/qubes-*.egg-info/*
@@ -221,6 +222,7 @@ fi
 
 %dir %{python_sitelib}/qubes/tools
 %{python_sitelib}/qubes/tools/__init__.py*
+%{python_sitelib}/qubes/tools/qmemmand.py*
 %{python_sitelib}/qubes/tools/qubes_create.py*
 %{python_sitelib}/qubes/tools/qvm_create.py*
 %{python_sitelib}/qubes/tools/qvm_ls.py*
@@ -249,15 +251,13 @@ fi
 %{python_sitelib}/qubes/tests/tools/init.py*
 %{python_sitelib}/qubes/tests/tools/qvm_ls.py*
 
-# qmemman
-%{python_sitelib}/qubes/qmemman.py*
-%{python_sitelib}/qubes/qmemman_algo.py*
-%{python_sitelib}/qubes/qmemman_client.py*
-%{python_sitelib}/qubes/qmemman_server.py*
+%dir %{python_sitelib}/qubes/qmemman
+%{python_sitelib}/qubes/qmemman/__init__.py*
+%{python_sitelib}/qubes/qmemman/algo.py*
+%{python_sitelib}/qubes/qmemman/client.py*
 
 /usr/lib/qubes/unbind-pci-device.sh
 /usr/lib/qubes/cleanup-dispvms
-/usr/lib/qubes/qmemman_daemon.py*
 /usr/lib/qubes/qfile-daemon-dvm*
 /usr/lib/qubes/block-cleaner-daemon.py*
 /usr/lib/qubes/vusb-ctl.py*