From 7f27d987cc6a9526f30c810b1812e06f5eaae623 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Thu, 13 Nov 2014 14:38:41 +0100 Subject: [PATCH 0001/1004] import framework for core3 --- qubes/__init__.py | 4 ++ qubes/_pluginloader.py | 6 +++ qubes/events.py | 34 ++++++++++++++++ qubes/ext/__init__.py | 36 +++++++++++++++++ qubes/plugins.py | 30 ++++++++++++++ qubes/vm/__init__.py | 89 +++++++++++++++++++++++++++++++++++++++++ qubes/vm/adminvm.py | 7 ++++ qubes/vm/appvm.py | 7 ++++ qubes/vm/dispvm.py | 7 ++++ qubes/vm/hvm.py | 7 ++++ qubes/vm/netvm.py | 7 ++++ qubes/vm/proxyvm.py | 7 ++++ qubes/vm/qubesvm.py | 7 ++++ qubes/vm/templatehvm.py | 7 ++++ qubes/vm/templatevm.py | 7 ++++ 15 files changed, 262 insertions(+) create mode 100644 qubes/__init__.py create mode 100644 qubes/_pluginloader.py create mode 100644 qubes/events.py create mode 100644 qubes/ext/__init__.py create mode 100644 qubes/plugins.py create mode 100644 qubes/vm/__init__.py create mode 100644 qubes/vm/adminvm.py create mode 100644 qubes/vm/appvm.py create mode 100644 qubes/vm/dispvm.py create mode 100644 qubes/vm/hvm.py create mode 100644 qubes/vm/netvm.py create mode 100644 qubes/vm/proxyvm.py create mode 100644 qubes/vm/qubesvm.py create mode 100644 qubes/vm/templatehvm.py create mode 100644 qubes/vm/templatevm.py diff --git a/qubes/__init__.py b/qubes/__init__.py new file mode 100644 index 00000000..307b3ff4 --- /dev/null +++ b/qubes/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/python2 -O + +import qubes._pluginloader + diff --git a/qubes/_pluginloader.py b/qubes/_pluginloader.py new file mode 100644 index 00000000..4c9c7a58 --- /dev/null +++ b/qubes/_pluginloader.py @@ -0,0 +1,6 @@ +from qubes.vm import * +from qubes.ext import * + +import qubes.ext + +qubes.ext.init() diff --git a/qubes/events.py b/qubes/events.py new file mode 100644 index 00000000..95f31162 --- /dev/null +++ b/qubes/events.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2 -O + +import collections + +import qubes.vm + +system_hooks = collections.defaultdict(list) + +def hook(event, vm=None, system=False): + def decorator(f): + f.ho_event = event + + if system: + f.ho_vm = None + elif vm is None: + f.ho_vm = qubes.vm.BaseVM + else: + f.ho_vm = vm + + return f + + return decorator + +def ishook(o): + return callable(o) \ + and hasattr(o, 'ho_event') \ + and hasattr(o, 'ho_vm') + +def add_system_hook(event, f): + global_hooks[event].append(f) + +def fire_system_hooks(event, *args, **kwargs): + for hook in system_hooks[event]: + hook(self, *args, **kwargs) diff --git a/qubes/ext/__init__.py b/qubes/ext/__init__.py new file mode 100644 index 00000000..bd794dd4 --- /dev/null +++ b/qubes/ext/__init__.py @@ -0,0 +1,36 @@ +#!/usr/bin/python2 -O + +import inspect + +import qubes.events +import qubes.plugins + +class ExtensionPlugin(qubes.plugins.Plugin): + def __init__(cls, name, bases, dict_): + super(ExtensionPlugin, cls).__init__(name, bases, dict_) + cls._instance = None + + def __call__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super(ExtensionPlugin, cls).__call__(*args, **kwargs) + return cls._instance + +class Extension(object): + __metaclass__ = ExtensionPlugin + def __init__(self): + for name in dir(self): + attr = getattr(self, name) + if not ishook(attr): + continue + + if attr.ho_vm is not None: + attr.ho_vm.add_hook(event, attr) + else: + # global hook + qubes.events.add_system_hook(event, attr) + +def init(): + for ext in Extension.register.values(): + instance = ext() + +__all__ = qubes.plugins.load(__file__) diff --git a/qubes/plugins.py b/qubes/plugins.py new file mode 100644 index 00000000..79059823 --- /dev/null +++ b/qubes/plugins.py @@ -0,0 +1,30 @@ +#!/usr/bin/python2 -O +# -*- coding: utf-8 -*- + +import imp +import inspect +import os +import sys + +class Plugin(type): + def __init__(cls, name, bases, dict_): + if hasattr(cls, 'register'): + cls.register[cls.__name__] = cls + else: + # we've got root class + cls.register = {} + + def __getitem__(cls, name): + return cls.register[name] + +def load(modfile): + path = os.path.dirname(modfile) + listdir = os.listdir(path) + ret = set() + for suffix, mode, type_ in imp.get_suffixes(): + for filename in listdir: + if filename.endswith(suffix): + ret.add(filename[:-len(suffix)]) + if '__init__' in ret: + ret.remove('__init__') + return list(sorted(ret)) diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py new file mode 100644 index 00000000..7025804d --- /dev/null +++ b/qubes/vm/__init__.py @@ -0,0 +1,89 @@ +#!/usr/bin/python2 -O + +import collections +import functools +import sys + +import dateutil.parser + +import qubes.plugins + +class property(object): + def __init__(self, name, default=None, type=None, order=0, doc=None): + self.__name__ = name + self._default = default + self._type = type + self.order = order + self.__doc__ = doc + + self._attr_name = '_qubesprop_' + self.__name__ + + def __get__(self, instance, owner): + if instance is None: + return self + + try: + return getattr(instance, self._attr_name) + + except AttributeError: + if self._default is None: + raise AttributeError('property not set') + else: + return self._default + + def __set__(self, instance, value): + setattr(instance, self._attr_name, + (self._type(value) if self._type is not None else value)) + + def __repr__(self): + return '<{} object at {:#x} name={!r} default={!r}>'.format( + self.__class__.__name__, id(self), self.__name__, self._default) + + def __hash__(self): + return hash(self.__name__) + + def __eq__(self, other): + return self.__name__ == other.__name__ + +class VMPlugin(qubes.plugins.Plugin): + def __init__(cls, name, bases, dict_): + super(VMPlugin, cls).__init__(name, bases, dict_) + cls.__hooks__ = collections.defaultdict(list) + +class BaseVM(object): + __metaclass__ = VMPlugin + + def get_props_list(self): + props = set() + for class_ in self.__class__.__mro__: + props.update(prop for prop in class_.__dict__.values() + if isinstance(prop, property)) + return sorted(props, key=lambda prop: (prop.order, prop.__name__)) + + def __init__(self, D): + for prop in self.get_props_list(): + if prop.__name__ in D: + setattr(self, prop.__name__, D[prop.__name__]) + + def __repr__(self): + return '<{} object at {:#x} {}>'.format( + self.__class__.__name__, id(self), + ' '.join('{}={}'.format(prop.__name__, getattr(self, prop.__name__)) + for prop in self.get_props_list())) + + @classmethod + def add_hook(cls, event, f): + cls.__hooks__[event].append(f) + + def fire_hooks(self, event, *args, **kwargs): + for cls in self.__class__.__mro__: + if not hasattr(cls, '__hooks__'): continue + for hook in cls.__hooks__[event]: + hook(self, *args, **kwargs) + + +def load(class_, D): + cls = BaseVM[class_] + return cls(D) + +__all__ = qubes.plugins.load(__file__) diff --git a/qubes/vm/adminvm.py b/qubes/vm/adminvm.py new file mode 100644 index 00000000..b0b0e1b1 --- /dev/null +++ b/qubes/vm/adminvm.py @@ -0,0 +1,7 @@ +#!/usr/bin/python2 -O + +import qubes.vm.netvm + +class AdminVM(qubes.vm.netvm.NetVM): + def __init__(self, D): + super(AdminVM, self).__init__(D) diff --git a/qubes/vm/appvm.py b/qubes/vm/appvm.py new file mode 100644 index 00000000..7296f9b2 --- /dev/null +++ b/qubes/vm/appvm.py @@ -0,0 +1,7 @@ +#!/usr/bin/python2 -O + +import qubes.vm.qubesvm + +class AppVM(qubes.vm.qubesvm.QubesVM): + def __init__(self, D): + super(AppVM, self).__init__(D) diff --git a/qubes/vm/dispvm.py b/qubes/vm/dispvm.py new file mode 100644 index 00000000..4adefac6 --- /dev/null +++ b/qubes/vm/dispvm.py @@ -0,0 +1,7 @@ +#!/usr/bin/python2 -O + +import qubes.vm.qubesvm + +class DispVM(qubes.vm.qubesvm.QubesVM): + def __init__(self, D): + super(DispVM, self).__init__(D) diff --git a/qubes/vm/hvm.py b/qubes/vm/hvm.py new file mode 100644 index 00000000..1a178b6a --- /dev/null +++ b/qubes/vm/hvm.py @@ -0,0 +1,7 @@ +#!/usr/bin/python2 -O + +import qubes.vm.qubesvm + +class HVM(qubes.vm.qubesvm.QubesVM): + def __init__(self, D): + super(HVM, self).__init__(D) diff --git a/qubes/vm/netvm.py b/qubes/vm/netvm.py new file mode 100644 index 00000000..45b79a26 --- /dev/null +++ b/qubes/vm/netvm.py @@ -0,0 +1,7 @@ +#!/usr/bin/python2 -O + +import qubes.vm.qubesvm + +class NetVM(qubes.vm.qubesvm.QubesVM): + def __init__(self, D): + super(NetVM, self).__init__(D) diff --git a/qubes/vm/proxyvm.py b/qubes/vm/proxyvm.py new file mode 100644 index 00000000..1033c899 --- /dev/null +++ b/qubes/vm/proxyvm.py @@ -0,0 +1,7 @@ +#!/usr/bin/python2 -O + +import qubes.vm.netvm + +class ProxyVM(qubes.vm.netvm.NetVM): + def __init__(self, D): + super(ProxyVM, self).__init__(D) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py new file mode 100644 index 00000000..8c28ebb5 --- /dev/null +++ b/qubes/vm/qubesvm.py @@ -0,0 +1,7 @@ +#!/usr/bin/python2 -O + +import qubes.vm + +class QubesVM(qubes.vm.BaseVM): + def __init__(self, D): + super(QubesVM, self).__init__(D) diff --git a/qubes/vm/templatehvm.py b/qubes/vm/templatehvm.py new file mode 100644 index 00000000..3cbc11bd --- /dev/null +++ b/qubes/vm/templatehvm.py @@ -0,0 +1,7 @@ +#!/usr/bin/python2 -O + +import qubes.vm.hvm + +class TemplateHVM(qubes.vm.hvm.HVM): + def __init__(self, D): + super(TemplateHVM, self).__init__(D) diff --git a/qubes/vm/templatevm.py b/qubes/vm/templatevm.py new file mode 100644 index 00000000..f71f31c7 --- /dev/null +++ b/qubes/vm/templatevm.py @@ -0,0 +1,7 @@ +#!/usr/bin/python2 -O + +import qubes.vm.qubesvm + +class TemplateVM(qubes.vm.qubesvm.QubesVM): + def __init__(self, D): + super(TemplateVM, self).__init__(D) From 65595e3b39bbdcb0b5200bf9438d61464942560c Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Thu, 13 Nov 2014 18:10:27 +0100 Subject: [PATCH 0002/1004] apidoc stub --- doc/apidoc/Makefile | 153 +++++++++++++++++ doc/apidoc/conf.py | 254 ++++++++++++++++++++++++++++ doc/apidoc/index.rst | 27 +++ doc/apidoc/qubes-events.rst | 8 + doc/apidoc/qubes-ext.rst | 6 + doc/apidoc/qubes-plugins.rst | 8 + doc/apidoc/qubes-vm/adminvm.rst | 8 + doc/apidoc/qubes-vm/appvm.rst | 8 + doc/apidoc/qubes-vm/dispvm.rst | 8 + doc/apidoc/qubes-vm/hvm.rst | 8 + doc/apidoc/qubes-vm/index.rst | 6 + doc/apidoc/qubes-vm/netvm.rst | 8 + doc/apidoc/qubes-vm/proxyvm.rst | 8 + doc/apidoc/qubes-vm/qubesvm.rst | 8 + doc/apidoc/qubes-vm/templatehvm.rst | 8 + doc/apidoc/qubes-vm/templatevm.rst | 8 + doc/apidoc/qubes.rst | 6 + qubes/__init__.py | 8 + qubes/events.py | 37 ++++ qubes/ext/__init__.py | 18 ++ qubes/plugins.py | 12 ++ qubes/vm/__init__.py | 81 +++++++++ qubes/vm/adminvm.py | 1 + qubes/vm/appvm.py | 1 + qubes/vm/dispvm.py | 1 + qubes/vm/hvm.py | 1 + qubes/vm/netvm.py | 1 + qubes/vm/proxyvm.py | 1 + qubes/vm/qubesvm.py | 1 + qubes/vm/templatehvm.py | 1 + qubes/vm/templatevm.py | 1 + 31 files changed, 705 insertions(+) create mode 100644 doc/apidoc/Makefile create mode 100644 doc/apidoc/conf.py create mode 100644 doc/apidoc/index.rst create mode 100644 doc/apidoc/qubes-events.rst create mode 100644 doc/apidoc/qubes-ext.rst create mode 100644 doc/apidoc/qubes-plugins.rst create mode 100644 doc/apidoc/qubes-vm/adminvm.rst create mode 100644 doc/apidoc/qubes-vm/appvm.rst create mode 100644 doc/apidoc/qubes-vm/dispvm.rst create mode 100644 doc/apidoc/qubes-vm/hvm.rst create mode 100644 doc/apidoc/qubes-vm/index.rst create mode 100644 doc/apidoc/qubes-vm/netvm.rst create mode 100644 doc/apidoc/qubes-vm/proxyvm.rst create mode 100644 doc/apidoc/qubes-vm/qubesvm.rst create mode 100644 doc/apidoc/qubes-vm/templatehvm.rst create mode 100644 doc/apidoc/qubes-vm/templatevm.rst create mode 100644 doc/apidoc/qubes.rst diff --git a/doc/apidoc/Makefile b/doc/apidoc/Makefile new file mode 100644 index 00000000..c46c8695 --- /dev/null +++ b/doc/apidoc/Makefile @@ -0,0 +1,153 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/core-admin.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/core-admin.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/core-admin" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/core-admin" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/apidoc/conf.py b/doc/apidoc/conf.py new file mode 100644 index 00000000..b01d733d --- /dev/null +++ b/doc/apidoc/conf.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +# +# core-admin documentation build configuration file, created by +# sphinx-quickstart on Thu Nov 13 15:02:15 2014. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import os +import subprocess +import sys +import time + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../../')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'core-admin' +copyright = u'2010-{}, Invisible Things Lab'.format(time.strftime('%Y')) + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = open('../../version').read() +# The full version, including alpha/beta/rc tags. +release = subprocess.check_output(['git', 'describe', '--long', '--dirty']).strip() + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +today_fmt = '%d.%m.%Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +autodoc_member_order = 'groupwise' + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +#html_theme = 'default' +html_theme = 'nature' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { +# 'collapsiblesidebar': True, +} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +html_last_updated_fmt = '%d.%m.%Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'core-admin-doc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'core-admin.tex', u'core-admin Documentation', + u'Invisible Things Lab', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'core-admin', u'core-admin Documentation', + [u'Invisible Things Lab'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'core-admin', u'core-admin Documentation', + u'Invisible Things Lab', 'core-admin', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + + +# Example configuration for intersphinx: refer to the Python standard library. +#intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/doc/apidoc/index.rst b/doc/apidoc/index.rst new file mode 100644 index 00000000..572c5eb8 --- /dev/null +++ b/doc/apidoc/index.rst @@ -0,0 +1,27 @@ +.. core-admin documentation master file, created by + sphinx-quickstart on Thu Nov 13 15:02:15 2014. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to core-admin's documentation! +====================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + qubes + qubes-vm/index + qubes-events + qubes-plugins + qubes-ext + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-events.rst b/doc/apidoc/qubes-events.rst new file mode 100644 index 00000000..941fe784 --- /dev/null +++ b/doc/apidoc/qubes-events.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.events` Qubes events +=================================== + +.. automodule:: qubes.events + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-ext.rst b/doc/apidoc/qubes-ext.rst new file mode 100644 index 00000000..ff71fe4e --- /dev/null +++ b/doc/apidoc/qubes-ext.rst @@ -0,0 +1,6 @@ +:py:mod:`qubes.ext` Qubes extensions +======================================== + +.. automodule:: qubes.ext + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-plugins.rst b/doc/apidoc/qubes-plugins.rst new file mode 100644 index 00000000..fa1630d2 --- /dev/null +++ b/doc/apidoc/qubes-plugins.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.plugins` Plugin helpers +====================================== + +.. automodule:: qubes.plugins + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-vm/adminvm.rst b/doc/apidoc/qubes-vm/adminvm.rst new file mode 100644 index 00000000..8f3edf7b --- /dev/null +++ b/doc/apidoc/qubes-vm/adminvm.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.vm.adminvm` Dom0 +=============================== + +.. automodule:: qubes.vm.adminvm + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-vm/appvm.rst b/doc/apidoc/qubes-vm/appvm.rst new file mode 100644 index 00000000..2f95be5b --- /dev/null +++ b/doc/apidoc/qubes-vm/appvm.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.vm.appvm` Application VM +======================================= + +.. automodule:: qubes.vm.appvm + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-vm/dispvm.rst b/doc/apidoc/qubes-vm/dispvm.rst new file mode 100644 index 00000000..f307e16f --- /dev/null +++ b/doc/apidoc/qubes-vm/dispvm.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.vm.dispvm` Disposable VM +======================================= + +.. automodule:: qubes.vm.dispvm + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-vm/hvm.rst b/doc/apidoc/qubes-vm/hvm.rst new file mode 100644 index 00000000..477c3fc8 --- /dev/null +++ b/doc/apidoc/qubes-vm/hvm.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.vm.hvm` HVM +========================== + +.. automodule:: qubes.vm.hvm + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-vm/index.rst b/doc/apidoc/qubes-vm/index.rst new file mode 100644 index 00000000..2d6733f3 --- /dev/null +++ b/doc/apidoc/qubes-vm/index.rst @@ -0,0 +1,6 @@ +:py:mod:`qubes.vm` Different Virtual Machine types +================================================== + +.. automodule:: qubes.vm + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-vm/netvm.rst b/doc/apidoc/qubes-vm/netvm.rst new file mode 100644 index 00000000..8ca90d4d --- /dev/null +++ b/doc/apidoc/qubes-vm/netvm.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.vm.netvm` Network interface VM +============================================= + +.. automodule:: qubes.vm.netvm + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-vm/proxyvm.rst b/doc/apidoc/qubes-vm/proxyvm.rst new file mode 100644 index 00000000..665f0ab9 --- /dev/null +++ b/doc/apidoc/qubes-vm/proxyvm.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.vm.proxyvm` Proxy (firewall/VPN) VM +================================================== + +.. automodule:: qubes.vm.proxyvm + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-vm/qubesvm.rst b/doc/apidoc/qubes-vm/qubesvm.rst new file mode 100644 index 00000000..f5119ea6 --- /dev/null +++ b/doc/apidoc/qubes-vm/qubesvm.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.vm.qubesvm` Shared functionality +=============================================== + +.. automodule:: qubes.vm.qubesvm + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-vm/templatehvm.rst b/doc/apidoc/qubes-vm/templatehvm.rst new file mode 100644 index 00000000..4f668220 --- /dev/null +++ b/doc/apidoc/qubes-vm/templatehvm.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.vm.templatehvm` Template for HVM +=============================================== + +.. automodule:: qubes.vm.templatehvm + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes-vm/templatevm.rst b/doc/apidoc/qubes-vm/templatevm.rst new file mode 100644 index 00000000..5842ada8 --- /dev/null +++ b/doc/apidoc/qubes-vm/templatevm.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.vm.templatevm` Template for AppVM +================================================ + +.. automodule:: qubes.vm.templatevm + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes.rst b/doc/apidoc/qubes.rst new file mode 100644 index 00000000..d886ddab --- /dev/null +++ b/doc/apidoc/qubes.rst @@ -0,0 +1,6 @@ +:py:mod:`qubes` Common concepts +=============================== + +.. automodule:: qubes + +.. vim: ts=3 sw=3 et diff --git a/qubes/__init__.py b/qubes/__init__.py index 307b3ff4..513b351b 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -1,4 +1,12 @@ #!/usr/bin/python2 -O +''' +Qubes OS +''' + +__author__ = 'Invisible Things Lab' +__license__ = 'GPLv2 or later' +__version__ = 'R3' + import qubes._pluginloader diff --git a/qubes/events.py b/qubes/events.py index 95f31162..e1fe13fb 100644 --- a/qubes/events.py +++ b/qubes/events.py @@ -1,12 +1,30 @@ #!/usr/bin/python2 -O +'''Qubes events. + +Events are fired when something happens, like VM start or stop, property change +etc. + +''' + import collections import qubes.vm +#: collection of system-wide hooks system_hooks = collections.defaultdict(list) def hook(event, vm=None, system=False): + '''Decorator factory. + + To hook an event, decorate a method in your plugin class with this + decorator. + + :param str event: event type + :param type vm: VM to hook (leave as None to hook all VMs) + :param bool system: when :py:obj:`True`, hook is system-wide (not attached to any VM) + ''' + def decorator(f): f.ho_event = event @@ -22,13 +40,32 @@ def hook(event, vm=None, system=False): return decorator def ishook(o): + '''Test if a method is hooked to an event. + + :param object o: suspected hook + :return: :py:obj:`True` when function is a hook, :py:obj:`False` otherwise + :rtype: bool + ''' + return callable(o) \ and hasattr(o, 'ho_event') \ and hasattr(o, 'ho_vm') def add_system_hook(event, f): + '''Add system-wide hook. + + :param callable f: function to call + ''' + global_hooks[event].append(f) def fire_system_hooks(event, *args, **kwargs): + '''Fire system-wide hooks. + + :param str event: event type + + *args* and *kwargs* are passed to all hooks. + ''' + for hook in system_hooks[event]: hook(self, *args, **kwargs) diff --git a/qubes/ext/__init__.py b/qubes/ext/__init__.py index bd794dd4..01353e78 100644 --- a/qubes/ext/__init__.py +++ b/qubes/ext/__init__.py @@ -1,11 +1,28 @@ #!/usr/bin/python2 -O +'''Qubes extensions + +Extensions provide additional features (like application menus) found only on +some systems. They may be OS- or architecture-dependent or custom-developed for +particular customer. + +.. autoclass:: Extension + :members: + :show-inheritance: + +.. autoclass:: ExtensionPlugin + :members: + :show-inheritance: + +''' + import inspect import qubes.events import qubes.plugins class ExtensionPlugin(qubes.plugins.Plugin): + '''Metaclass for :py:class:`Extension`''' def __init__(cls, name, bases, dict_): super(ExtensionPlugin, cls).__init__(name, bases, dict_) cls._instance = None @@ -16,6 +33,7 @@ class ExtensionPlugin(qubes.plugins.Plugin): return cls._instance class Extension(object): + '''Base class for all extensions''' __metaclass__ = ExtensionPlugin def __init__(self): for name in dir(self): diff --git a/qubes/plugins.py b/qubes/plugins.py index 79059823..970a87f6 100644 --- a/qubes/plugins.py +++ b/qubes/plugins.py @@ -1,12 +1,18 @@ #!/usr/bin/python2 -O # -*- coding: utf-8 -*- +'''Plugins helpers for Qubes + +Qubes uses two types of plugins: virtual machines and extensions. +''' + import imp import inspect import os import sys class Plugin(type): + '''Base metaclass for plugins''' def __init__(cls, name, bases, dict_): if hasattr(cls, 'register'): cls.register[cls.__name__] = cls @@ -18,6 +24,12 @@ class Plugin(type): return cls.register[name] def load(modfile): + '''Load (import) all plugins from subpackage. + + This function should be invoked from ``__init__.py`` in a package like that: + + >>> __all__ = qubes.plugins.load(__file__) # doctest: +SKIP + ''' path = os.path.dirname(modfile) listdir = os.listdir(path) ret = set() diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index 7025804d..52dca286 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -1,5 +1,56 @@ #!/usr/bin/python2 -O +'''Qubes Virtual Machines + +Main public classes +------------------- + +.. autoclass:: BaseVM + :members: + :show-inheritance: +.. autoclass:: property + :members: + :show-inheritance: + +Helper classes and functions +---------------------------- + +.. autoclass:: VMPlugin + :members: + :show-inheritance: + +Particular VM classes +--------------------- + +Main types: + +.. toctree:: + :maxdepth: 1 + + qubesvm + appvm + templatevm + +Special VM types: + +.. toctree:: + :maxdepth: 1 + + netvm + proxyvm + dispvm + adminvm + +HVMs: + +.. toctree:: + :maxdepth: 1 + + hvm + templatehvm + +''' + import collections import functools import sys @@ -9,6 +60,18 @@ import dateutil.parser import qubes.plugins class property(object): + '''Qubes VM property. + + This class holds one property that can be saved and loaded from qubes.xml + + :param str name: name of the property + :param object default: default value + :param type type: if not :py:obj:`None`, this is used to initialise value + :param int order: order of evaluation (bigger order values are later) + :param str doc: docstring + + ''' + def __init__(self, name, default=None, type=None, order=0, doc=None): self.__name__ = name self._default = default @@ -46,6 +109,7 @@ class property(object): return self.__name__ == other.__name__ class VMPlugin(qubes.plugins.Plugin): + '''Metaclass for :py:class:`.BaseVM`''' def __init__(cls, name, bases, dict_): super(VMPlugin, cls).__init__(name, bases, dict_) cls.__hooks__ = collections.defaultdict(list) @@ -54,6 +118,7 @@ class BaseVM(object): __metaclass__ = VMPlugin def get_props_list(self): + '''List all properties attached to this VM''' props = set() for class_ in self.__class__.__mro__: props.update(prop for prop in class_.__dict__.values() @@ -73,9 +138,25 @@ class BaseVM(object): @classmethod def add_hook(cls, event, f): + '''Add hook to entire VM class and all subclasses + + :param str event: event type + :param callable f: function to fire on event + + Prototype of the function depends on the exact type of event. Classes + which inherit from this class will also inherit the hook. + ''' + cls.__hooks__[event].append(f) def fire_hooks(self, event, *args, **kwargs): + '''Fire hooks associated with an event. + + :param str event: event type + + *args* and *kwargs* are passed to each function + ''' + for cls in self.__class__.__mro__: if not hasattr(cls, '__hooks__'): continue for hook in cls.__hooks__[event]: diff --git a/qubes/vm/adminvm.py b/qubes/vm/adminvm.py index b0b0e1b1..ef7c8284 100644 --- a/qubes/vm/adminvm.py +++ b/qubes/vm/adminvm.py @@ -3,5 +3,6 @@ import qubes.vm.netvm class AdminVM(qubes.vm.netvm.NetVM): + '''Dom0''' def __init__(self, D): super(AdminVM, self).__init__(D) diff --git a/qubes/vm/appvm.py b/qubes/vm/appvm.py index 7296f9b2..8d58c319 100644 --- a/qubes/vm/appvm.py +++ b/qubes/vm/appvm.py @@ -3,5 +3,6 @@ import qubes.vm.qubesvm class AppVM(qubes.vm.qubesvm.QubesVM): + '''Application VM''' def __init__(self, D): super(AppVM, self).__init__(D) diff --git a/qubes/vm/dispvm.py b/qubes/vm/dispvm.py index 4adefac6..90553de5 100644 --- a/qubes/vm/dispvm.py +++ b/qubes/vm/dispvm.py @@ -3,5 +3,6 @@ import qubes.vm.qubesvm class DispVM(qubes.vm.qubesvm.QubesVM): + '''Disposable VM''' def __init__(self, D): super(DispVM, self).__init__(D) diff --git a/qubes/vm/hvm.py b/qubes/vm/hvm.py index 1a178b6a..9bf970b3 100644 --- a/qubes/vm/hvm.py +++ b/qubes/vm/hvm.py @@ -3,5 +3,6 @@ import qubes.vm.qubesvm class HVM(qubes.vm.qubesvm.QubesVM): + '''HVM''' def __init__(self, D): super(HVM, self).__init__(D) diff --git a/qubes/vm/netvm.py b/qubes/vm/netvm.py index 45b79a26..2e84e5d8 100644 --- a/qubes/vm/netvm.py +++ b/qubes/vm/netvm.py @@ -3,5 +3,6 @@ import qubes.vm.qubesvm class NetVM(qubes.vm.qubesvm.QubesVM): + '''Network interface VM''' def __init__(self, D): super(NetVM, self).__init__(D) diff --git a/qubes/vm/proxyvm.py b/qubes/vm/proxyvm.py index 1033c899..9924d0f7 100644 --- a/qubes/vm/proxyvm.py +++ b/qubes/vm/proxyvm.py @@ -3,5 +3,6 @@ import qubes.vm.netvm class ProxyVM(qubes.vm.netvm.NetVM): + '''Proxy (firewall/VPN) VM''' def __init__(self, D): super(ProxyVM, self).__init__(D) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 8c28ebb5..95b030dc 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -3,5 +3,6 @@ import qubes.vm class QubesVM(qubes.vm.BaseVM): + '''Base functionality of Qubes VM shared between all VMs.''' def __init__(self, D): super(QubesVM, self).__init__(D) diff --git a/qubes/vm/templatehvm.py b/qubes/vm/templatehvm.py index 3cbc11bd..5be33733 100644 --- a/qubes/vm/templatehvm.py +++ b/qubes/vm/templatehvm.py @@ -3,5 +3,6 @@ import qubes.vm.hvm class TemplateHVM(qubes.vm.hvm.HVM): + '''Template for HVM''' def __init__(self, D): super(TemplateHVM, self).__init__(D) diff --git a/qubes/vm/templatevm.py b/qubes/vm/templatevm.py index f71f31c7..d1df2495 100644 --- a/qubes/vm/templatevm.py +++ b/qubes/vm/templatevm.py @@ -3,5 +3,6 @@ import qubes.vm.qubesvm class TemplateVM(qubes.vm.qubesvm.QubesVM): + '''Template for AppVM''' def __init__(self, D): super(TemplateVM, self).__init__(D) From e1a6fb2859b6b92ae7155c926b57e5d369995f7b Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Fri, 14 Nov 2014 15:41:27 +0100 Subject: [PATCH 0003/1004] core3 move: class QubesException --- core/qubes.py | 3 --- qubes/__init__.py | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/qubes.py b/core/qubes.py index 8f814709..107c285f 100755 --- a/core/qubes.py +++ b/core/qubes.py @@ -125,9 +125,6 @@ defaults = { qubes_max_qid = 254 qubes_max_netid = 254 -class QubesException (Exception): - pass - class QubesVMMConnection(object): def __init__(self): self._libvirt_conn = None diff --git a/qubes/__init__.py b/qubes/__init__.py index 513b351b..db4b724d 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -10,3 +10,7 @@ __version__ = 'R3' import qubes._pluginloader +class QubesException(Exception): + '''Exception that can be shown to the user''' + pass + From cec3db993db8c37720f1af789c8249c89ac079a7 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Fri, 14 Nov 2014 15:41:27 +0100 Subject: [PATCH 0004/1004] core3 move: class QubesVMMConnection --- core/qubes.py | 64 ------------------------------------------- qubes/__init__.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 64 deletions(-) diff --git a/core/qubes.py b/core/qubes.py index 107c285f..9ab73cf4 100755 --- a/core/qubes.py +++ b/core/qubes.py @@ -125,70 +125,6 @@ defaults = { qubes_max_qid = 254 qubes_max_netid = 254 -class QubesVMMConnection(object): - def __init__(self): - self._libvirt_conn = None - self._xs = None - self._xc = None - self._offline_mode = False - - @property - def offline_mode(self): - return self._offline_mode - - @offline_mode.setter - def offline_mode(self, value): - if not value and self._libvirt_conn is not None: - raise QubesException("Cannot change offline mode while already connected") - - self._offline_mode = value - - def _libvirt_error_handler(self, ctx, error): - pass - - def init_vmm_connection(self): - if self._libvirt_conn is not None: - # Already initialized - return - if self._offline_mode: - # Do not initialize in offline mode - return - - if 'xen.lowlevel.xs' in sys.modules: - self._xs = xen.lowlevel.xs.xs() - libvirt.virEventRegisterDefaultImpl() - self._libvirt_conn = libvirt.open(defaults['libvirt_uri']) - if self._libvirt_conn == None: - raise QubesException("Failed connect to libvirt driver") - libvirt.registerErrorHandler(self._libvirt_error_handler, None) - atexit.register(self._libvirt_conn.close) - - def _common_getter(self, name): - if self._offline_mode: - # Do not initialize in offline mode - raise QubesException("VMM operations disabled in offline mode") - - if self._libvirt_conn is None: - self.init_vmm_connection() - return getattr(self, name) - - @property - def libvirt_conn(self): - return self._common_getter('_libvirt_conn') - - @property - def xs(self): - if 'xen.lowlevel.xs' in sys.modules: - return self._common_getter('_xs') - else: - return None - - -##### VMM global variable definition ##### - -if not dry_run: - vmm = QubesVMMConnection() - ########################################## class QubesHost(object): diff --git a/qubes/__init__.py b/qubes/__init__.py index db4b724d..3cefbd78 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -14,3 +14,73 @@ class QubesException(Exception): '''Exception that can be shown to the user''' pass +class QubesVMMConnection(object): + '''Connection to Virtual Machine Manager (libvirt)''' + def __init__(self): + self._libvirt_conn = None + self._xs = None + self._xc = None + self._offline_mode = False + + @property + def offline_mode(self): + '''Check or enable offline mode (do not actually connect to vmm)''' + return self._offline_mode + + @offline_mode.setter + def offline_mode(self, value): + if not value and self._libvirt_conn is not None: + raise QubesException("Cannot change offline mode while already connected") + + self._offline_mode = value + + def _libvirt_error_handler(self, ctx, error): + pass + + def init_vmm_connection(self): + '''Initialise connection + + This method is automatically called when getting''' + if self._libvirt_conn is not None: + # Already initialized + return + if self._offline_mode: + # Do not initialize in offline mode + return + + if 'xen.lowlevel.xs' in sys.modules: + self._xs = xen.lowlevel.xs.xs() + self._libvirt_conn = libvirt.open(defaults['libvirt_uri']) + if self._libvirt_conn == None: + raise QubesException("Failed connect to libvirt driver") + libvirt.registerErrorHandler(self._libvirt_error_handler, None) + atexit.register(self._libvirt_conn.close) + + def _common_getter(self, name): + if self._offline_mode: + # Do not initialize in offline mode + raise QubesException("VMM operations disabled in offline mode") + + if self._libvirt_conn is None: + self.init_vmm_connection() + return getattr(self, name) + + @property + def libvirt_conn(self): + '''Connection to libvirt''' + return self._common_getter('_libvirt_conn') + + @property + def xs(self): + '''Connection to Xen Store + + This property in available only when running on Xen.''' + + if 'xen.lowlevel.xs' in sys.modules: + return self._common_getter('_xs') + else: + return None + +if not dry_run: + vmm = QubesVMMConnection() + From 57d35fbc4c58b52db9ea23d1fd0ae8645deed452 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 17 Nov 2014 13:46:53 +0100 Subject: [PATCH 0005/1004] qubes: fix qubes.QubesVMMConnection fix logical error in resetting offline_mode and drop redundant _common_getter() --- qubes/__init__.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index 3cefbd78..8f9a6112 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -29,7 +29,7 @@ class QubesVMMConnection(object): @offline_mode.setter def offline_mode(self, value): - if not value and self._libvirt_conn is not None: + if value and self._libvirt_conn is not None: raise QubesException("Cannot change offline mode while already connected") self._offline_mode = value @@ -46,7 +46,7 @@ class QubesVMMConnection(object): return if self._offline_mode: # Do not initialize in offline mode - return + raise QubesException("VMM operations disabled in offline mode") if 'xen.lowlevel.xs' in sys.modules: self._xs = xen.lowlevel.xs.xs() @@ -56,19 +56,11 @@ class QubesVMMConnection(object): libvirt.registerErrorHandler(self._libvirt_error_handler, None) atexit.register(self._libvirt_conn.close) - def _common_getter(self, name): - if self._offline_mode: - # Do not initialize in offline mode - raise QubesException("VMM operations disabled in offline mode") - - if self._libvirt_conn is None: - self.init_vmm_connection() - return getattr(self, name) - @property def libvirt_conn(self): '''Connection to libvirt''' - return self._common_getter('_libvirt_conn') + self.init_vmm_connection() + return self._libvirt_conn @property def xs(self): @@ -76,11 +68,12 @@ class QubesVMMConnection(object): This property in available only when running on Xen.''' - if 'xen.lowlevel.xs' in sys.modules: - return self._common_getter('_xs') - else: + if 'xen.lowlevel.xs' not in sys.modules: return None + self.init_vmm_connection() + return self._xs + if not dry_run: vmm = QubesVMMConnection() From 320cb096f6a9286d6945513f20a662a8c20db6b6 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 17 Nov 2014 13:52:02 +0100 Subject: [PATCH 0006/1004] qubes: drop dry_run --- qubes/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index 8f9a6112..74d21dcb 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -74,6 +74,5 @@ class QubesVMMConnection(object): self.init_vmm_connection() return self._xs -if not dry_run: - vmm = QubesVMMConnection() +vmm = QubesVMMConnection() From 1fbb91a1aafb11649db34ea550435f2876c6c40f Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 17 Nov 2014 16:23:33 +0100 Subject: [PATCH 0007/1004] doc/apidoc: enable intersphinx to docs.python.org --- doc/apidoc/conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/apidoc/conf.py b/doc/apidoc/conf.py index b01d733d..8399d849 100644 --- a/doc/apidoc/conf.py +++ b/doc/apidoc/conf.py @@ -29,7 +29,7 @@ sys.path.insert(0, os.path.abspath('../../')) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. #extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -251,4 +251,5 @@ texinfo_documents = [ # Example configuration for intersphinx: refer to the Python standard library. -#intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = { + 'python': ('http://docs.python.org/', None)} From c3dd13c0ab836b1f97e6e62f2cd75ba0e1ef5737 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 17 Nov 2014 16:25:14 +0100 Subject: [PATCH 0008/1004] qubes/log: logging routines --- doc/apidoc/index.rst | 1 + doc/apidoc/qubes-log.rst | 8 +++++ doc/apidoc/qubes.rst | 2 ++ qubes/log.py | 70 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+) create mode 100644 doc/apidoc/qubes-log.rst create mode 100644 qubes/log.py diff --git a/doc/apidoc/index.rst b/doc/apidoc/index.rst index 572c5eb8..4b1aa224 100644 --- a/doc/apidoc/index.rst +++ b/doc/apidoc/index.rst @@ -16,6 +16,7 @@ Contents: qubes-events qubes-plugins qubes-ext + qubes-log Indices and tables ================== diff --git a/doc/apidoc/qubes-log.rst b/doc/apidoc/qubes-log.rst new file mode 100644 index 00000000..f6fa3e7e --- /dev/null +++ b/doc/apidoc/qubes-log.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.log` Logging routines +==================================== + +.. automodule:: qubes.log + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/doc/apidoc/qubes.rst b/doc/apidoc/qubes.rst index d886ddab..e2ae68b3 100644 --- a/doc/apidoc/qubes.rst +++ b/doc/apidoc/qubes.rst @@ -2,5 +2,7 @@ =============================== .. automodule:: qubes + :members: + :show-inheritance: .. vim: ts=3 sw=3 et diff --git a/qubes/log.py b/qubes/log.py new file mode 100644 index 00000000..1e43f748 --- /dev/null +++ b/qubes/log.py @@ -0,0 +1,70 @@ +#!/usr/bin/python2 -O +# -*- coding: utf-8 -*- + +'''Qubes logging routines + +See also: :py:attr:`qubes.vm.qubesvm.QubesVM.logger` +''' + +import logging +import os +import sys + +FORMAT_CONSOLE = '%(message)s' +FORMAT_LOG = '%(asctime)s %(message)s' +FORMAT_DEBUG = '%(asctime)s [%(processName)s %(module)s.%(funcName)s:%(lineno)d] %(name)s: %(message)s' +LOGPATH = '/var/log/qubes' +LOGFILE = os.path.join(LOGPATH, 'qubes.log') + +formatter_console = logging.Formatter(FORMAT_CONSOLE) +formatter_log = logging.Formatter(FORMAT_LOG) +formatter_debug = logging.Formatter(FORMAT_DEBUG) + +def enable(): + '''Enable global logging + + Use :py:mod:`logging` module from standard library to log messages. + + >>> import qubes.log + >>> qubes.log.enable() # doctest: +SKIP + >>> import logging + >>> logging.warning('Foobar') # doctest: +SKIP + ''' + if logging.root.handlers: + return + + handler_console = logging.StreamHandler(sys.stderr) + handler_console.setFormatter(formatter_console) + logging.root.addHandler(handler_console) + + handler_log = logging.FileHandler('log', 'a', encoding='utf-8') + handler_log.setFormatter(formatter_log) + logging.root.addHandler(handler_log) + + logging.root.setLevel(logging.INFO) + +def enable_debug(): + '''Enable debug logging + + Enable more messages and additional info to message format. + ''' + + enable() + logging.root.setLevel(logging.DEBUG) + + for handler in logging.root.handlers: + handler.setFormatter(formatter_debug) + +def get_vm_logger(vmname): + '''Initialise logging for particular VM name + + :param str vmname: VM's name + :rtype: :py:class:`logging.Logger` + ''' + + logger = logging.getLogger('vm.' + vmname) + handler = logging.FileHandler(os.path.join(LOGPATH, 'vm', vmname + '.log')) + handler.setFormatter(formatter_log) + logger.addHandler(handler) + + return logger From 778571fe8d1f091dc8078bd667d9d0fe9686aa20 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 17 Nov 2014 17:07:08 +0100 Subject: [PATCH 0009/1004] core3 move: class QubesHost --- core/qubes.py | 56 ------------------------------------------- qubes/__init__.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 56 deletions(-) diff --git a/core/qubes.py b/core/qubes.py index 9ab73cf4..f58991f7 100755 --- a/core/qubes.py +++ b/core/qubes.py @@ -127,62 +127,6 @@ qubes_max_netid = 254 ########################################## -class QubesHost(object): - def __init__(self): - (model, memory, cpus, mhz, nodes, socket, cores, threads) = vmm.libvirt_conn.getInfo() - self._total_mem = long(memory)*1024 - self._no_cpus = cpus - -# print "QubesHost: total_mem = {0}B".format (self.xen_total_mem) -# print "QubesHost: free_mem = {0}".format (self.get_free_xen_memory()) -# print "QubesHost: total_cpus = {0}".format (self.xen_no_cpus) - - @property - def memory_total(self): - return self._total_mem - - @property - def no_cpus(self): - return self._no_cpus - - # TODO - def measure_cpu_usage(self, qvmc, previous=None, previous_time = None, - wait_time=1): - """measure cpu usage for all domains at once""" - if previous is None: - previous_time = time.time() - previous = {} - for vm in qvmc.values(): - if not vm.is_running(): - continue - cputime = vm.get_cputime() - previous[vm.xid] = {} - previous[vm.xid]['cpu_time'] = ( - cputime / vm.vcpus) - previous[vm.xid]['cpu_usage'] = 0 - time.sleep(wait_time) - - current_time = time.time() - current = {} - for vm in qvmc.values(): - if not vm.is_running(): - continue - cputime = vm.get_cputime() - current[vm.xid] = {} - current[vm.xid]['cpu_time'] = ( - cputime / max(vm.vcpus, 1)) - if vm.xid in previous.keys(): - current[vm.xid]['cpu_usage'] = ( - float(current[vm.xid]['cpu_time'] - - previous[vm.xid]['cpu_time']) / - long(1000**3) / (current_time-previous_time) * 100) - if current[vm.xid]['cpu_usage'] < 0: - # VM has been rebooted - current[vm.xid]['cpu_usage'] = 0 - else: - current[vm.xid]['cpu_usage'] = 0 - - return (current_time, current) class QubesVmLabel(object): def __init__(self, index, color, name, dispvm=False): diff --git a/qubes/__init__.py b/qubes/__init__.py index 74d21dcb..4f71e820 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -76,3 +76,64 @@ class QubesVMMConnection(object): vmm = QubesVMMConnection() + +class QubesHost(object): + '''Basic information about host machine''' + def __init__(self): + (model, memory, cpus, mhz, nodes, socket, cores, threads) = vmm.libvirt_conn.getInfo() + self._total_mem = long(memory)*1024 + self._no_cpus = cpus + +# print "QubesHost: total_mem = {0}B".format (self.xen_total_mem) +# print "QubesHost: free_mem = {0}".format (self.get_free_xen_memory()) +# print "QubesHost: total_cpus = {0}".format (self.xen_no_cpus) + + @property + def memory_total(self): + '''Total memory, in bytes''' + return self._total_mem + + @property + def no_cpus(self): + '''Noumber of CPUs''' + return self._no_cpus + + # TODO + def get_free_xen_memory(self): + ret = self.physinfo['free_memory'] + return long(ret) + + # TODO + def measure_cpu_usage(self, previous=None, previous_time = None, + wait_time=1): + """measure cpu usage for all domains at once""" + if previous is None: + previous_time = time.time() + previous = {} + info = vmm.xc.domain_getinfo(0, qubes_max_qid) + for vm in info: + previous[vm['domid']] = {} + previous[vm['domid']]['cpu_time'] = ( + vm['cpu_time'] / vm['online_vcpus']) + previous[vm['domid']]['cpu_usage'] = 0 + time.sleep(wait_time) + + current_time = time.time() + current = {} + info = vmm.xc.domain_getinfo(0, qubes_max_qid) + for vm in info: + current[vm['domid']] = {} + current[vm['domid']]['cpu_time'] = ( + vm['cpu_time'] / max(vm['online_vcpus'], 1)) + if vm['domid'] in previous.keys(): + current[vm['domid']]['cpu_usage'] = ( + float(current[vm['domid']]['cpu_time'] - + previous[vm['domid']]['cpu_time']) / + long(1000**3) / (current_time-previous_time) * 100) + if current[vm['domid']]['cpu_usage'] < 0: + # VM has been rebooted + current[vm['domid']]['cpu_usage'] = 0 + else: + current[vm['domid']]['cpu_usage'] = 0 + + return (current_time, current) From f3673dd34c67e72963b63de42c7b7ef79227344f Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 17 Nov 2014 19:09:25 +0100 Subject: [PATCH 0010/1004] core3 move: class QubesVmLabel --- core/qubes.py | 46 ------------------------------ qubes/__init__.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 46 deletions(-) diff --git a/core/qubes.py b/core/qubes.py index f58991f7..574b4894 100755 --- a/core/qubes.py +++ b/core/qubes.py @@ -128,29 +128,6 @@ qubes_max_netid = 254 ########################################## -class QubesVmLabel(object): - def __init__(self, index, color, name, dispvm=False): - self.index = index - self.color = color - self.name = name - self.dispvm = dispvm - - self.icon = '{}-{}'.format(('dispvm' if dispvm else 'appvm'), name) - - def __repr__(self): - return '{}({!r}, {!r}, {!r}, dispvm={!r})'.format( - self.__class__.__name__, - self.index, - self.color, - self.name, - self.dispvm) - - # self.icon_path is obsolete - # use QIcon.fromTheme(label.icon) where applicable - @property - def icon_path(self): - return os.path.join(system_path['qubes_icon_dir'], self.icon) + ".png" - def register_qubes_vm_class(vm_class): QubesVmClasses[vm_class.__name__] = vm_class # register class as local for this module - to make it easy to import from @@ -793,29 +770,6 @@ class QubesDaemonPidfile(object): ### Initialization code -# Globally defined lables -QubesVmLabels = { - "red": QubesVmLabel(1, "0xcc0000", "red" ), - "orange": QubesVmLabel(2, "0xf57900", "orange" ), - "yellow": QubesVmLabel(3, "0xedd400", "yellow" ), - "green": QubesVmLabel(4, "0x73d216", "green" ), - "gray": QubesVmLabel(5, "0x555753", "gray" ), - "blue": QubesVmLabel(6, "0x3465a4", "blue" ), - "purple": QubesVmLabel(7, "0x75507b", "purple" ), - "black": QubesVmLabel(8, "0x000000", "black" ), -} - -QubesDispVmLabels = { - "red": QubesVmLabel(1, "0xcc0000", "red", dispvm=True), - "orange": QubesVmLabel(2, "0xf57900", "orange", dispvm=True), - "yellow": QubesVmLabel(3, "0xedd400", "yellow", dispvm=True), - "green": QubesVmLabel(4, "0x73d216", "green", dispvm=True), - "gray": QubesVmLabel(5, "0x555753", "gray", dispvm=True), - "blue": QubesVmLabel(6, "0x3465a4", "blue", dispvm=True), - "purple": QubesVmLabel(7, "0x75507b", "purple", dispvm=True), - "black": QubesVmLabel(8, "0x000000", "black", dispvm=True), -} - defaults["appvm_label"] = QubesVmLabels["red"] defaults["template_label"] = QubesVmLabels["black"] defaults["servicevm_label"] = QubesVmLabels["red"] diff --git a/qubes/__init__.py b/qubes/__init__.py index 4f71e820..adf0deec 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -137,3 +137,75 @@ class QubesHost(object): current[vm['domid']]['cpu_usage'] = 0 return (current_time, current) + + +class QubesVmLabel(object): + '''Label definition for virtual machines + + Label specifies colour of the padlock displayed next to VM's name. + When this is a :py:class:`qubes.vm.dispvm.DispVM`, padlock is overlayed + with recycling pictogram. + + :param int index: numeric identificator of label + :param str color: colour specification as in HTML (``#abcdef``) + :param str name: label's name like "red" or "green" + :param bool dispvm: :py:obj:`True` if this is :py:class:`qubes.vm.dispvm.DispVM` label + + ''' + def __init__(self, index, color, name, dispvm=False): + #: numeric identificator of label + self.index = index + + #: colour specification as in HTML (``#abcdef``) + self.color = color + + #: label's name like "red" or "green" + self.name = name + + #: :py:obj:`True` if this is :py:class:`qubes.vm.dispvm.DispVM` label + self.dispvm = dispvm + + #: freedesktop icon name, suitable for use in :py:meth:`PyQt4.QtGui.QIcon.fromTheme` + self.icon = '{}-{}'.format(('dispvm' if dispvm else 'appvm'), name) + + def __repr__(self): + return '{}({!r}, {!r}, {!r}, dispvm={!r})'.format( + self.__class__.__name__, + self.index, + self.color, + self.name, + self.dispvm) + + # self.icon_path is obsolete + # use QIcon.fromTheme(label.icon) where applicable + @property + def icon_path(self): + '''Icon path + + DEPRECATED --- use :py:meth:`PyQt4.QtGui.QIcon.fromTheme` and :py:attr:`QubesVmLabel.icon`''' + return os.path.join(system_path['qubes_icon_dir'], self.icon) + ".png" + +#: Globally defined labels +QubesVmLabels = { + "red": QubesVmLabel(1, "0xcc0000", "red" ), + "orange": QubesVmLabel(2, "0xf57900", "orange" ), + "yellow": QubesVmLabel(3, "0xedd400", "yellow" ), + "green": QubesVmLabel(4, "0x73d216", "green" ), + "gray": QubesVmLabel(5, "0x555753", "gray" ), + "blue": QubesVmLabel(6, "0x3465a4", "blue" ), + "purple": QubesVmLabel(7, "0x75507b", "purple" ), + "black": QubesVmLabel(8, "0x000000", "black" ), +} + +#: Globally defined labels for :py:class:`qubes.vm.dispvm.DispVM` s +QubesDispVmLabels = { + "red": QubesVmLabel(1, "0xcc0000", "red", dispvm=True), + "orange": QubesVmLabel(2, "0xf57900", "orange", dispvm=True), + "yellow": QubesVmLabel(3, "0xedd400", "yellow", dispvm=True), + "green": QubesVmLabel(4, "0x73d216", "green", dispvm=True), + "gray": QubesVmLabel(5, "0x555753", "gray", dispvm=True), + "blue": QubesVmLabel(6, "0x3465a4", "blue", dispvm=True), + "purple": QubesVmLabel(7, "0x75507b", "purple", dispvm=True), + "black": QubesVmLabel(8, "0x000000", "black", dispvm=True), +} + From 87ae0112eb3e6516cd3180aa6751afb6773a11b6 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Tue, 18 Nov 2014 17:35:05 +0100 Subject: [PATCH 0011/1004] qubes/vm: New XML format loading --- qubes/vm/__init__.py | 56 +++++++++++++++++++++++++--- tests/vm.py | 88 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 tests/vm.py diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index 52dca286..dc7423a5 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -51,6 +51,7 @@ HVMs: ''' +import ast import collections import functools import sys @@ -115,6 +116,16 @@ class VMPlugin(qubes.plugins.Plugin): cls.__hooks__ = collections.defaultdict(list) class BaseVM(object): + '''Base class for all VMs + + :param xml: xml node from which to deserialise + :type xml: :py:class:`lxml.etree._Element` or :py:obj:`None` + + This class is responsible for serialising and deserialising machines and + provides basic framework. It contains no management logic. For that, see + :py:class:`qubes.vm.qubesvm.QubesVM`. + ''' + __metaclass__ = VMPlugin def get_props_list(self): @@ -125,10 +136,45 @@ class BaseVM(object): if isinstance(prop, property)) return sorted(props, key=lambda prop: (prop.order, prop.__name__)) - def __init__(self, D): - for prop in self.get_props_list(): - if prop.__name__ in D: - setattr(self, prop.__name__, D[prop.__name__]) + def __init__(self, xml): + self._xml = xml + + self.services = {} + self.devices = collections.defaultdict(list) + self.tags = {} + + if self._xml is None: + return + + # properties + all_names = set(prop.__name__ for prop in self.get_props_list()) + for node in self._xml.xpath('.//property'): + name = node.get('name') + value = node.get('ref') or node.text + + if not name in all_names: + raise AttributeError( + 'No property {!r} found in {!r}'.format( + name, self.__class__)) + + setattr(self, name, value) + + # tags + for node in self._xml.xpath('.//tag'): + self.tags[node.get('name')] = node.text + + # services + for node in self._xml.xpath('.//service'): + self.services[node.text] = bool(ast.literal_eval(node.get('enabled', 'True'))) + + # devices (pci, usb, ...) + for parent in self._xml.xpath('.//devices'): + devclass = parent.get('class') + for node in parent.xpath('./device'): + self.devices[devclass].append(node.text) + + # firewall + #TODO def __repr__(self): return '<{} object at {:#x} {}>'.format( @@ -150,7 +196,7 @@ class BaseVM(object): cls.__hooks__[event].append(f) def fire_hooks(self, event, *args, **kwargs): - '''Fire hooks associated with an event. + '''Fire hooks associated with an event :param str event: event type diff --git a/tests/vm.py b/tests/vm.py new file mode 100644 index 00000000..273875e8 --- /dev/null +++ b/tests/vm.py @@ -0,0 +1,88 @@ +#!/usr/bin/python2 -O + +import sys +import unittest + +import lxml.etree + +sys.path.insert(0, '../') +import qubes.vm + +class TestVM(qubes.vm.BaseVM): + testprop = qubes.vm.property('testprop') + testlabel = qubes.vm.property('testlabel') + defaultprop = qubes.vm.property('defaultprop', default='defaultvalue') + +class TC_BaseVM(unittest.TestCase): + def setUp(self): + self.xml = lxml.etree.XML(''' + + + + + + + + + testvalue + + + + + tagvalue + + + + testservice + enabledservice + disabledservice + + + + 00:11.22 + + + + + + + + + + + ''') + + def test_000_BaseVM_load(self): + node = self.xml.xpath('//domain')[0] + vm = TestVM(node) + + self.assertEqual(vm.testprop, 'testvalue') + self.assertEqual(vm.testlabel, 'label-1') + self.assertEqual(vm.defaultprop, 'defaultvalue') + self.assertEqual(vm.tags, {'testtag': 'tagvalue'}) + self.assertEqual(vm.devices, {'pci': ['00:11.22']}) + self.assertEqual(vm.services, { + 'testservice': True, + 'enabledservice': True, + 'disabledservice': False, + }) + + def test_001_BaseVM_nxproperty(self): + xml = lxml.etree.XML(''' + + + + + nxvalue + + + + + ''') + + node = xml.xpath('//domain')[0] + + def f(): + vm = TestVM(node) + + self.assertRaises(AttributeError, f) From 96bff66546916dd319386b5682772c9be2ed2258 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Thu, 20 Nov 2014 15:20:49 +0100 Subject: [PATCH 0012/1004] qubes.dochelpers: Helpers for Sphinx documentation Currently it is possible to refer to Qubes' tickets via :ticket:`#no` --- doc/apidoc/conf.py | 3 +- doc/apidoc/index.rst | 1 + doc/apidoc/qubes-dochelpers.rst | 8 ++++ qubes/dochelpers.py | 76 +++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 doc/apidoc/qubes-dochelpers.rst create mode 100644 qubes/dochelpers.py diff --git a/doc/apidoc/conf.py b/doc/apidoc/conf.py index 8399d849..3a05d7ff 100644 --- a/doc/apidoc/conf.py +++ b/doc/apidoc/conf.py @@ -28,8 +28,7 @@ sys.path.insert(0, os.path.abspath('../../')) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'qubes.dochelpers'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/doc/apidoc/index.rst b/doc/apidoc/index.rst index 4b1aa224..ec19628b 100644 --- a/doc/apidoc/index.rst +++ b/doc/apidoc/index.rst @@ -17,6 +17,7 @@ Contents: qubes-plugins qubes-ext qubes-log + qubes-dochelpers Indices and tables ================== diff --git a/doc/apidoc/qubes-dochelpers.rst b/doc/apidoc/qubes-dochelpers.rst new file mode 100644 index 00000000..6f1c9ae1 --- /dev/null +++ b/doc/apidoc/qubes-dochelpers.rst @@ -0,0 +1,8 @@ +:py:mod:`qubes.dochelpers` Helpers for Sphinx documentation +=========================================================== + +.. automodule:: qubes.dochelpers + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et diff --git a/qubes/dochelpers.py b/qubes/dochelpers.py new file mode 100644 index 00000000..95a333e1 --- /dev/null +++ b/qubes/dochelpers.py @@ -0,0 +1,76 @@ +#!/usr/bin/python2 -O +# -*- coding: utf-8 -*- + +'''Documentation helpers + +This module contains classes and functions which help to mainain documentation, +particulary our custom Sphinx extension. + +''' + +import csv +import posixpath +import sys +import urllib2 + +import docutils +import docutils.parsers.rst.roles + +def fetch_ticket_info(uri): + '''Fetch info about particular trac ticket given + + :param str uri: URI at which ticket resides + :rtype: mapping + :raises: urllib2.HTTPError + ''' + + data = urllib2.urlopen(uri + '?format=csv').read() + reader = csv.reader((line + '\n' for line in data.split('\r\n')), + quoting=csv.QUOTE_MINIMAL, quotechar='"') + + return dict(zip(*((cell.decode('utf-8') for cell in row) for row in list(reader)[:2]))) + + +def ticket(name, rawtext, text, lineno, inliner, options={}, content=[]): + '''Link to qubes ticket + + :param str name: The role name used in the document + :param str rawtext: The entire markup snippet, with role + :param str text: The text marked with the role + :param int lineno: The line noumber where rawtext appearn in the input + :param docutils.parsers.rst.states.Inliner inliner: The inliner instance that called this function + :param options: Directive options for customisation + :param content: The directive content for customisation + ''' + + ticket = text.lstrip('#') + if not ticket.isdigit(): + msg = inliner.reporter.error('Invalid ticket identificator: {!r}'.format(text), line=lineno) + prb = inliner.problematic(rawtext, rawtext, msg) + return [prb], [msg] + + app = inliner.document.settings.env.app + uri = posixpath.join(app.config.ticket_base_uri, ticket) + try: + info = fetch_ticket_info(uri) + except urllib2.HTTPError, e: + msg = inliner.reporter.error('Error while fetching ticket info: {!s}'.format(e), line=lineno) + prb = inliner.problematic(rawtext, rawtext, msg) + return [prb], [msg] + + docutils.parsers.rst.roles.set_classes(options) + + node = docutils.nodes.reference( + rawtext, + '#{} ({})'.format(ticket, info['summary']), + refuri=uri, + **options) + + return [node], [] + + +def setup(app): + app.add_role('ticket', ticket) + app.add_config_value('ticket_base_uri', 'https://wiki.qubes-os.org/ticket/', 'env') + +# vim: ts=4 sw=4 et From 2c1cacc0acfafcac4d6ffa200908d0649a3f571e Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Fri, 21 Nov 2014 12:30:23 +0100 Subject: [PATCH 0013/1004] doc: swallow manpages into sphinx --- doc/Makefile | 184 +++++++++++++++--- doc/apidoc/Makefile | 153 --------------- doc/{apidoc => }/conf.py | 61 +++++- doc/{apidoc => }/index.rst | 17 +- doc/{apidoc => }/qubes-dochelpers.rst | 0 doc/{apidoc => }/qubes-events.rst | 0 doc/{apidoc => }/qubes-ext.rst | 0 doc/{apidoc => }/qubes-log.rst | 0 doc/{apidoc => }/qubes-plugins.rst | 0 doc/qubes-tools/index.rst | 8 + .../{qubes_guid.rst => qubes-guid.rst} | 26 +-- doc/qubes-tools/qubes-prefs.rst | 16 +- doc/{apidoc => }/qubes-vm/adminvm.rst | 0 doc/{apidoc => }/qubes-vm/appvm.rst | 0 doc/{apidoc => }/qubes-vm/dispvm.rst | 0 doc/{apidoc => }/qubes-vm/hvm.rst | 0 doc/{apidoc => }/qubes-vm/index.rst | 0 doc/{apidoc => }/qubes-vm/netvm.rst | 0 doc/{apidoc => }/qubes-vm/proxyvm.rst | 0 doc/{apidoc => }/qubes-vm/qubesvm.rst | 0 doc/{apidoc => }/qubes-vm/templatehvm.rst | 0 doc/{apidoc => }/qubes-vm/templatevm.rst | 0 doc/{apidoc => }/qubes.rst | 0 doc/qvm-tools/index.rst | 8 + doc/qvm-tools/qvm-add-appvm.rst | 39 ++-- doc/qvm-tools/qvm-add-template.rst | 34 ++-- doc/qvm-tools/qvm-backup-restore.rst | 57 ++++-- doc/qvm-tools/qvm-backup.rst | 26 +-- doc/qvm-tools/qvm-block.rst | 47 +++-- doc/qvm-tools/qvm-clone-template.rst | 29 +-- doc/qvm-tools/qvm-clone.rst | 30 +-- doc/qvm-tools/qvm-create-default-dvm.rst | 17 +- doc/qvm-tools/qvm-create.rst | 86 +++++--- doc/qvm-tools/qvm-firewall.rst | 76 +++++--- doc/qvm-tools/qvm-grow-private.rst | 22 +-- doc/qvm-tools/qvm-kill.rst | 22 +-- doc/qvm-tools/qvm-ls.rst | 54 +++-- doc/qvm-tools/qvm-pci.rst | 35 ++-- doc/qvm-tools/qvm-prefs.rst | 36 ++-- doc/qvm-tools/qvm-remove.rst | 34 ++-- doc/qvm-tools/qvm-revert-template-changes.rst | 26 +-- doc/qvm-tools/qvm-run.rst | 92 ++++++--- doc/qvm-tools/qvm-service.rst | 45 +++-- doc/qvm-tools/qvm-shutdown.rst | 54 ++--- doc/qvm-tools/qvm-start.rst | 42 ++-- doc/qvm-tools/qvm-template-commit.rst | 22 +-- rpm_spec/core-dom0-doc.spec | 2 +- 47 files changed, 805 insertions(+), 595 deletions(-) delete mode 100644 doc/apidoc/Makefile rename doc/{apidoc => }/conf.py (72%) rename doc/{apidoc => }/index.rst (59%) rename doc/{apidoc => }/qubes-dochelpers.rst (100%) rename doc/{apidoc => }/qubes-events.rst (100%) rename doc/{apidoc => }/qubes-ext.rst (100%) rename doc/{apidoc => }/qubes-log.rst (100%) rename doc/{apidoc => }/qubes-plugins.rst (100%) create mode 100644 doc/qubes-tools/index.rst rename doc/qubes-tools/{qubes_guid.rst => qubes-guid.rst} (53%) rename doc/{apidoc => }/qubes-vm/adminvm.rst (100%) rename doc/{apidoc => }/qubes-vm/appvm.rst (100%) rename doc/{apidoc => }/qubes-vm/dispvm.rst (100%) rename doc/{apidoc => }/qubes-vm/hvm.rst (100%) rename doc/{apidoc => }/qubes-vm/index.rst (100%) rename doc/{apidoc => }/qubes-vm/netvm.rst (100%) rename doc/{apidoc => }/qubes-vm/proxyvm.rst (100%) rename doc/{apidoc => }/qubes-vm/qubesvm.rst (100%) rename doc/{apidoc => }/qubes-vm/templatehvm.rst (100%) rename doc/{apidoc => }/qubes-vm/templatevm.rst (100%) rename doc/{apidoc => }/qubes.rst (100%) create mode 100644 doc/qvm-tools/index.rst diff --git a/doc/Makefile b/doc/Makefile index a8aec59c..95b72af8 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -1,30 +1,164 @@ -QVM_DIR=qvm-tools -QUBES_DIR=qubes-tools -PANDOC=pandoc -s -f rst -t man +# Makefile for Sphinx documentation +# -QVM_DOCS=$(patsubst %.rst,%.1.gz,$(wildcard $(QVM_DIR)/*.rst)) -QUBES_DOCS=$(patsubst %.rst,%.1.gz,$(wildcard $(QUBES_DIR)/*.rst)) +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: - @echo "make rst=example.rst preview -- generate manpage preview from example.rst" - @echo "make manpages -- generate manpages" - @echo "make install -- generate manpages and copy them to /usr/share/man" - -install: manpages - mkdir -p $(DESTDIR)/usr/share/man/man1 - cp $(QVM_DOCS) $(DESTDIR)/usr/share/man/man1/ - cp $(QUBES_DOCS) $(DESTDIR)/usr/share/man/man1/ - -%.1: %.rst - $(PANDOC) $< > $@ - -%.1.gz: %.1 - gzip -f $< - -manpages: $(QVM_DOCS) $(QUBES_DOCS) $(VM_DOCS) - -preview: $(rst) - pandoc -s -f rst -t man $(rst) | groff -mandoc -Tlatin1 | less -R + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo + @echo " install to generate manpages and copy them to \$$(DESTDIR)/usr/share/man" clean: - rm -f $(QVM_DOCS) $(QUBES_DOCS) $(VM_DOCS) + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/core-admin.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/core-admin.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/core-admin" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/core-admin" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + for file in $(BUILDDIR)/man/*.[12345678]; do \ + gzip -f $$file; \ + done + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + + +.PHONY: install +install: man + mkdir -p $(DESTDIR)/usr/share/man/man1 + cp $(BUILDDIR)/man/* $(DESTDIR)/usr/share/man/man1/ diff --git a/doc/apidoc/Makefile b/doc/apidoc/Makefile deleted file mode 100644 index c46c8695..00000000 --- a/doc/apidoc/Makefile +++ /dev/null @@ -1,153 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/core-admin.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/core-admin.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/core-admin" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/core-admin" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/apidoc/conf.py b/doc/conf.py similarity index 72% rename from doc/apidoc/conf.py rename to doc/conf.py index 3a05d7ff..1a34519e 100644 --- a/doc/apidoc/conf.py +++ b/doc/conf.py @@ -19,7 +19,7 @@ import time # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath('../')) # -- General configuration ----------------------------------------------------- @@ -51,7 +51,7 @@ copyright = u'2010-{}, Invisible Things Lab'.format(time.strftime('%Y')) # built documents. # # The short X.Y version. -version = open('../../version').read() +version = open('../version').read().strip() # The full version, including alpha/beta/rc tags. release = subprocess.check_output(['git', 'describe', '--long', '--dirty']).strip() @@ -219,9 +219,62 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). + +# authors should be empty and authors should be specified in each man page, +# because html builder will omit them +_man_pages_author = [] + man_pages = [ - ('index', 'core-admin', u'core-admin Documentation', - [u'Invisible Things Lab'], 1) + ('qvm-tools/qvm-add-appvm', 'qvm-add-appvm', + u'Add an already installed appvm to the Qubes DB', _man_pages_author, 1), + ('qvm-tools/qvm-add-template', 'qvm-add-template', + u'Adds an already installed template to the Qubes DB', _man_pages_author, 1), + ('qvm-tools/qvm-backup-restore', 'qvm-backup-restore', + u'Restores Qubes VMs from backup', _man_pages_author, 1), + ('qvm-tools/qvm-backup', 'qvm-backup', + u'Create backup of specified VMs', _man_pages_author, 1), + ('qvm-tools/qvm-block', 'qvm-block', + u'List/set VM block devices.', _man_pages_author, 1), + ('qvm-tools/qvm-clone-template', 'qvm-clone-template', + u'Clones an existing template by copying all its disk files', _man_pages_author, 1), + ('qvm-tools/qvm-clone', 'qvm-clone', + u'Clones an existing VM by copying all its disk files', _man_pages_author, 1), + ('qvm-tools/qvm-create-default-dvm', 'qvm-create-default-dvm', + u'Creates a default disposable VM', _man_pages_author, 1), + ('qvm-tools/qvm-create', 'qvm-create', + u'Creates a new VM', _man_pages_author, 1), + ('qvm-tools/qvm-firewall', 'qvm-firewall', + u'Qubes firewall configuration', _man_pages_author, 1), + ('qvm-tools/qvm-grow-private', 'qvm-grow-private', + u'Increase private storage capacity of a specified VM', _man_pages_author, 1), + ('qvm-tools/qvm-kill', 'qvm-kill', + u'Kill the specified VM', _man_pages_author, 1), + ('qvm-tools/qvm-ls', 'qvm-ls', + u'List VMs and various information about them', _man_pages_author, 1), + ('qvm-tools/qvm-pci', 'qvm-pci', + u'List/set VM PCI devices', _man_pages_author, 1), + ('qvm-tools/qvm-prefs', 'qvm-prefs', + u'List/set various per-VM properties', _man_pages_author, 1), + ('qvm-tools/qvm-remove', 'qvm-remove', + u'Remove a VM', _man_pages_author, 1), + ('qvm-tools/qvm-revert-template-changes', 'qvm-revert-template-changes', + u'Revert changes to a template', _man_pages_author, 1), + ('qvm-tools/qvm-run', 'qvm-run', + u'Run a command on a specified VM', _man_pages_author, 1), + ('qvm-tools/qvm-service', 'qvm-service', + u'Manage (Qubes-specific) services started in VM', _man_pages_author, 1), + ('qvm-tools/qvm-shutdown', 'qvm-shutdown', + u'Gracefully shut down a VM', _man_pages_author, 1), + ('qvm-tools/qvm-start', 'qvm-start', + u'Start a specified VM', _man_pages_author, 1), + ('qvm-tools/qvm-template-commit', 'qvm-template-commit', + u'Commit changes to a template', _man_pages_author, 1), + + + ('qubes-tools/qubes-guid', 'qubes-guid', + u'Daemon for Qubes GUI isolation protocol', _man_pages_author, 1), + ('qubes-tools/qubes-prefs', 'qubes-prefs', + u'Display system-wide Qubes settings', _man_pages_author, 1), ] # If true, show URL addresses after external links. diff --git a/doc/apidoc/index.rst b/doc/index.rst similarity index 59% rename from doc/apidoc/index.rst rename to doc/index.rst index ec19628b..525a3d00 100644 --- a/doc/apidoc/index.rst +++ b/doc/index.rst @@ -6,7 +6,22 @@ Welcome to core-admin's documentation! ====================================== -Contents: +This page contains documentation autogenerated from source tree. It includes +manpages and API documentation. For primary user documentation, see +`https://wiki.qubes-os.org `_. + +User documentation +------------------ + +.. toctree:: + :maxdepth: 2 + :glob: + + qvm-tools/index + qubes-tools/index + +Developer documentation +----------------------- .. toctree:: :maxdepth: 2 diff --git a/doc/apidoc/qubes-dochelpers.rst b/doc/qubes-dochelpers.rst similarity index 100% rename from doc/apidoc/qubes-dochelpers.rst rename to doc/qubes-dochelpers.rst diff --git a/doc/apidoc/qubes-events.rst b/doc/qubes-events.rst similarity index 100% rename from doc/apidoc/qubes-events.rst rename to doc/qubes-events.rst diff --git a/doc/apidoc/qubes-ext.rst b/doc/qubes-ext.rst similarity index 100% rename from doc/apidoc/qubes-ext.rst rename to doc/qubes-ext.rst diff --git a/doc/apidoc/qubes-log.rst b/doc/qubes-log.rst similarity index 100% rename from doc/apidoc/qubes-log.rst rename to doc/qubes-log.rst diff --git a/doc/apidoc/qubes-plugins.rst b/doc/qubes-plugins.rst similarity index 100% rename from doc/apidoc/qubes-plugins.rst rename to doc/qubes-plugins.rst diff --git a/doc/qubes-tools/index.rst b/doc/qubes-tools/index.rst new file mode 100644 index 00000000..429b0bfa --- /dev/null +++ b/doc/qubes-tools/index.rst @@ -0,0 +1,8 @@ +System-wide tools +================= + +.. toctree:: + :maxdepth: 1 + :glob: + + * diff --git a/doc/qubes-tools/qubes_guid.rst b/doc/qubes-tools/qubes-guid.rst similarity index 53% rename from doc/qubes-tools/qubes_guid.rst rename to doc/qubes-tools/qubes-guid.rst index d1b35790..61c6e4d8 100644 --- a/doc/qubes-tools/qubes_guid.rst +++ b/doc/qubes-tools/qubes-guid.rst @@ -1,22 +1,22 @@ -========== -qubes_guid -========== +.. program:: qubes-guid -NAME -==== -qubes_guid +================================================================ +:program:`qubes-guid` -- Daemon for Qubes GUI isolation protocol +================================================================ -:Date: 2012-04-13 - -SYNOPSIS +Synopsis ======== -| qubes_guid -d domain_id [-c color] [-l label_index] [-i icon name, no suffix] [-v] [-q] +| qubes-guid -d domain_id [-c color] [-l label_index] [-i icon name, no suffix] [-v] [-q] -OPTIONS +Options ======= --v + +.. option:: -v + Increase log verbosity --q + +.. option:: -q + Decrease log verbosity Log levels: diff --git a/doc/qubes-tools/qubes-prefs.rst b/doc/qubes-tools/qubes-prefs.rst index 0d4b6ca4..ca0a4eb3 100644 --- a/doc/qubes-tools/qubes-prefs.rst +++ b/doc/qubes-tools/qubes-prefs.rst @@ -1,10 +1,8 @@ -=========== -qubes-prefs -=========== +.. program:: qubes-prefs -NAME -==== -qubes-prefs - display system-wide Qubes settings, such as: +============================================================ +:program:`qubes-prefs` -- Display system-wide Qubes settings +============================================================ - clock VM - update VM @@ -13,13 +11,11 @@ qubes-prefs - display system-wide Qubes settings, such as: - default kernel - default netVM -:Date: 2012-04-13 - -SYNOPSIS +Synopsis ======== | qubes-prefs -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/apidoc/qubes-vm/adminvm.rst b/doc/qubes-vm/adminvm.rst similarity index 100% rename from doc/apidoc/qubes-vm/adminvm.rst rename to doc/qubes-vm/adminvm.rst diff --git a/doc/apidoc/qubes-vm/appvm.rst b/doc/qubes-vm/appvm.rst similarity index 100% rename from doc/apidoc/qubes-vm/appvm.rst rename to doc/qubes-vm/appvm.rst diff --git a/doc/apidoc/qubes-vm/dispvm.rst b/doc/qubes-vm/dispvm.rst similarity index 100% rename from doc/apidoc/qubes-vm/dispvm.rst rename to doc/qubes-vm/dispvm.rst diff --git a/doc/apidoc/qubes-vm/hvm.rst b/doc/qubes-vm/hvm.rst similarity index 100% rename from doc/apidoc/qubes-vm/hvm.rst rename to doc/qubes-vm/hvm.rst diff --git a/doc/apidoc/qubes-vm/index.rst b/doc/qubes-vm/index.rst similarity index 100% rename from doc/apidoc/qubes-vm/index.rst rename to doc/qubes-vm/index.rst diff --git a/doc/apidoc/qubes-vm/netvm.rst b/doc/qubes-vm/netvm.rst similarity index 100% rename from doc/apidoc/qubes-vm/netvm.rst rename to doc/qubes-vm/netvm.rst diff --git a/doc/apidoc/qubes-vm/proxyvm.rst b/doc/qubes-vm/proxyvm.rst similarity index 100% rename from doc/apidoc/qubes-vm/proxyvm.rst rename to doc/qubes-vm/proxyvm.rst diff --git a/doc/apidoc/qubes-vm/qubesvm.rst b/doc/qubes-vm/qubesvm.rst similarity index 100% rename from doc/apidoc/qubes-vm/qubesvm.rst rename to doc/qubes-vm/qubesvm.rst diff --git a/doc/apidoc/qubes-vm/templatehvm.rst b/doc/qubes-vm/templatehvm.rst similarity index 100% rename from doc/apidoc/qubes-vm/templatehvm.rst rename to doc/qubes-vm/templatehvm.rst diff --git a/doc/apidoc/qubes-vm/templatevm.rst b/doc/qubes-vm/templatevm.rst similarity index 100% rename from doc/apidoc/qubes-vm/templatevm.rst rename to doc/qubes-vm/templatevm.rst diff --git a/doc/apidoc/qubes.rst b/doc/qubes.rst similarity index 100% rename from doc/apidoc/qubes.rst rename to doc/qubes.rst diff --git a/doc/qvm-tools/index.rst b/doc/qvm-tools/index.rst new file mode 100644 index 00000000..6fb256c0 --- /dev/null +++ b/doc/qvm-tools/index.rst @@ -0,0 +1,8 @@ +VM manipulation tools +===================== + +.. toctree:: + :maxdepth: 1 + :glob: + + * diff --git a/doc/qvm-tools/qvm-add-appvm.rst b/doc/qvm-tools/qvm-add-appvm.rst index 5fefab79..3ada1c79 100644 --- a/doc/qvm-tools/qvm-add-appvm.rst +++ b/doc/qvm-tools/qvm-add-appvm.rst @@ -1,29 +1,32 @@ -============= -qvm-add-appvm -============= +.. program:: qvm-add-appvm -NAME -==== -qvm-add-appvm - add an already installed appvm to the Qubes DB +========================================================================== +:program:`qvm-add-appvm` -- Add an already installed appvm to the Qubes DB +========================================================================== -WARNING: Noramlly you would not need this command, and you would use qvm-create instead! +.. warning:: + Normally you would not need this command, and you would use qvm-create instead! -:Date: 2012-04-10 - -SYNOPSIS +Synopsis ======== | qvm-add-appvm [options] -OPTIONS +Options ======= --h, --help - Show this help message and exit --p DIR_PATH, --path=DIR_PATH - Specify path to the template directory --c CONF_FILE, --conf=CONF_FILE - Specify the Xen VM .conf file to use(relative to the template dir path) -AUTHORS +.. option:: --help, -h + + Show this help message and exit + +.. option:: --path=DIR_PATH, -p DIR_PATH + + Specify path to the template directory + +.. option:: --conf=CONF_FILE, -c CONF_FILE + + Specify the Xen VM .conf file to use (relative to the template dir path) + +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-add-template.rst b/doc/qvm-tools/qvm-add-template.rst index 21961d58..1bf889f0 100644 --- a/doc/qvm-tools/qvm-add-template.rst +++ b/doc/qvm-tools/qvm-add-template.rst @@ -1,29 +1,33 @@ -================ -qvm-add-template -================ +.. program:: qvm-add-template -NAME -==== -qvm-add-template - adds an already installed template to the Qubes DB +================================================================================= +:program:`qvm-add-template` -- Adds an already installed template to the Qubes DB +================================================================================= -:Date: 2012-04-10 - -SYNOPSIS +Synopsis ======== | qvm-add-template [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit --p DIR_PATH, --path=DIR_PATH + +.. option:: --path=DIR_PATH, -p DIR_PATH + Specify path to the template directory --c CONF_FILE, --conf=CONF_FILE + +.. option:: --conf=CONF_FILE, -c CONF_FILE + Specify the Xen VM .conf file to use(relative to the template dir path) ---rpm + +.. option:: --rpm + Template files have been installed by RPM -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-backup-restore.rst b/doc/qvm-tools/qvm-backup-restore.rst index e078f7fa..c2f6db75 100644 --- a/doc/qvm-tools/qvm-backup-restore.rst +++ b/doc/qvm-tools/qvm-backup-restore.rst @@ -1,39 +1,54 @@ -================== -qvm-backup-restore -================== +.. program:: qvm-backup-restore -NAME -==== -qvm-backup-restore - restores Qubes VMs from backup +=============================================================== +:program:`qvm-backup-restore` -- Restores Qubes VMs from backup +=============================================================== -:Date: 2012-04-10 - -SYNOPSIS +Synopsis ======== | qvm-backup-restore [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit ---skip-broken + +.. option:: --skip-broken + Do not restore VMs that have missing templates or netvms ---ignore-missing + +.. option:: --ignore-missing + Ignore missing templates or netvms, restore VMs anyway ---skip-conflicting + +.. option:: --skip-conflicting + Do not restore VMs that are already present on the host ---force-root + +.. option:: --force-root + Force to run, even with root privileges ---replace-template=REPLACE_TEMPLATE - Restore VMs using another template, syntax: old-template-name:new-template-name (might be repeated) --x EXCLUDE, --exclude=EXCLUDE + +.. option:: --replace-template=REPLACE_TEMPLATE + + Restore VMs using another template, syntax: + ``old-template-name:new-template-name`` (might be repeated) + +.. option:: --exclude=EXCLUDE, -x EXCLUDE + Skip restore of specified VM (might be repeated) ---skip-dom0-home + +.. option:: --skip-dom0-home + Do not restore dom0 user home dir ---ignore-username-mismatch + +.. option:: --ignore-username-mismatch + Ignore dom0 username mismatch while restoring homedir -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-backup.rst b/doc/qvm-tools/qvm-backup.rst index 9d4dba8e..a0987caa 100644 --- a/doc/qvm-tools/qvm-backup.rst +++ b/doc/qvm-tools/qvm-backup.rst @@ -1,25 +1,25 @@ -========== -qvm-backup -========== +.. program:: qvm-backup -NAME -==== -qvm-backup +======================================================= +:program:`qvm-backup` -- Create backup of specified VMs +======================================================= -:Date: 2012-04-10 - -SYNOPSIS +Synopsis ======== | qvm-backup [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit --x EXCLUDE_LIST, --exclude=EXCLUDE_LIST + +.. option:: --exclude=EXCLUDE_LIST, -x EXCLUDE_LIST + Exclude the specified VM from backup (might be repeated) -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-block.rst b/doc/qvm-tools/qvm-block.rst index 839cfc6d..d637709e 100644 --- a/doc/qvm-tools/qvm-block.rst +++ b/doc/qvm-tools/qvm-block.rst @@ -1,15 +1,10 @@ -========= -qvm-block -========= +.. program:: qvm-block -NAME -==== -qvm-block - list/set VM PCI devices. +=============================================== +:program:`qvm-block` -- List/set VM PCI devices +=============================================== - -:Date: 2012-04-10 - -SYNOPSIS +Synopsis ======== | qvm-block -l [options] | qvm-block -a [options] @@ -17,24 +12,38 @@ SYNOPSIS | qvm-block -d [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit --l, --list + +.. option:: --list, -l + List block devices --a, --attach + +.. option:: --attach, -a + Attach block device to specified VM --d, --detach + +.. option:: --detach, -d + Detach block device --f FRONTEND, --frontend=FRONTEND + +.. option:: --frontend=FRONTEND, -f FRONTEND + Specify device name at destination VM [default: xvdi] ---ro + +.. option:: --ro + Force read-only mode ---no-auto-detach + +.. option:: --no-auto-detach + Fail when device already connected to other VM -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-clone-template.rst b/doc/qvm-tools/qvm-clone-template.rst index b7808059..112181e4 100644 --- a/doc/qvm-tools/qvm-clone-template.rst +++ b/doc/qvm-tools/qvm-clone-template.rst @@ -1,27 +1,28 @@ -================== -qvm-clone-template -================== +.. program:: qvm-clone-template -NAME -==== -qvm-clone-template - clones an existing template by copying all its disk files +========================================================================================== +:program:`qvm-clone-template` -- Clones an existing template by copying all its disk files +========================================================================================== -:Date: 2012-04-10 - -SYNOPSIS +Synopsis ======== | qvm-clone-template [options] -OPTIONS +Options ======= --h, --help +.. option:: --help, -h + Show this help message and exit --q, --quiet + +.. option:: --quiet, -q + Be quiet --p DIR_PATH, --path=DIR_PATH + +.. option:: --path=DIR_PATH, -p DIR_PATH + Specify path to the template directory -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-clone.rst b/doc/qvm-tools/qvm-clone.rst index 548a683f..d75682af 100644 --- a/doc/qvm-tools/qvm-clone.rst +++ b/doc/qvm-tools/qvm-clone.rst @@ -1,27 +1,29 @@ -========= -qvm-clone -========= +.. program:: qvm-clone -NAME -==== -qvm-clone - clones an existing VM by copying all its disk files +=========================================================================== +:program:`qvm-clone` -- Clones an existing VM by copying all its disk files +=========================================================================== -:Date: 2012-04-10 - -SYNOPSIS +Synopsis ======== | qvm-clone [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit --q, --quiet + +.. option:: --quiet, -q + Be quiet --p DIR_PATH, --path=DIR_PATH + +.. option:: --path=DIR_PATH, -p DIR_PATH + Specify path to the template directory -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-create-default-dvm.rst b/doc/qvm-tools/qvm-create-default-dvm.rst index b63fdb6f..046fb768 100644 --- a/doc/qvm-tools/qvm-create-default-dvm.rst +++ b/doc/qvm-tools/qvm-create-default-dvm.rst @@ -1,14 +1,10 @@ -====================== -qvm-create-default-dvm -====================== +.. program:: qvm-create-default-dvm -NAME -==== -qvm-create-default-dvm - creates a default disposable VM +==================================================================== +:program:`qvm-create-default-dvm` -- Creates a default Disposable VM +==================================================================== -:Date: 2012-04-10 - -SYNOPSIS +Synopsis ======== | qvm-create-default-dvm templatename|--default-template|--used-template [script-name|--default-script] @@ -29,8 +25,7 @@ templatename --default-script Use default script for seeding DispVM home. - -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-create.rst b/doc/qvm-tools/qvm-create.rst index 8da05414..8c54428a 100644 --- a/doc/qvm-tools/qvm-create.rst +++ b/doc/qvm-tools/qvm-create.rst @@ -1,53 +1,79 @@ -========== -qvm-create -========== +.. program:: qvm-create -NAME -==== -qvm-create - creates a new VM +========================================= +:program:`qvm-create` -- Creates a new VM +========================================= -:Date: 2012-04-10 - -SYNOPSIS +Synopsis ======== | qvm-create [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit --t TEMPLATE, --template=TEMPLATE + +.. option:: --template=TEMPLATE, -t TEMPLATE + Specify the TemplateVM to use --l LABEL, --label=LABEL + +.. option:: --label=LABEL, -l LABEL + Specify the label to use for the new VM (e.g. red, yellow, green, ...) --p, --proxy + +.. option:: --proxy, -p + Create ProxyVM --n, --net + +.. option:: --net, -n + Create NetVM --H, --hvm - Create HVM (standalone, unless --template option used) ---hvm-template + +.. option:: --hvm, -H + + Create HVM (standalone, unless :option:`--template` option used) + +.. option:: --hvm-template + Create HVM template --R ROOT_MOVE, --root-move-from=ROOT_MOVE + +.. option:: --root-move-from=ROOT_MOVE, -R ROOT_MOVE + Use provided root.img instead of default/empty one - (file will be MOVED) --r ROOT_COPY, --root-copy-from=ROOT_COPY + (file will be *moved*) + +.. option:: --root-copy-from=ROOT_COPY, -r ROOT_COPY + Use provided root.img instead of default/empty one - (file will be COPIED) --s, --standalone - Create standalone VM - independent of template --m MEM, --mem=MEM + (file will be *copied*) + +.. option:: --standalone, -s + + Create standalone VM --- independent of template + +.. option:: --mem=MEM, -m MEM + Initial memory size (in MB) --c VCPUS, --vcpus=VCPUS + +.. option:: --vcpus=VCPUS, -c VCPUS + VCPUs count --i, --internal + +.. option:: --internal, -i + Create VM for internal use only (hidden in qubes-manager, no appmenus) ---force-root + +.. option:: --force-root + Force to run, even with root privileges --q, --quiet + +.. option:: --quiet, -q + Be quiet -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-firewall.rst b/doc/qvm-tools/qvm-firewall.rst index 903da725..53837e90 100644 --- a/doc/qvm-tools/qvm-firewall.rst +++ b/doc/qvm-tools/qvm-firewall.rst @@ -1,14 +1,10 @@ -============ -qvm-firewall -============ +.. program:: qvm-firewall -NAME -==== -qvm-firewall +======================================================= +:program:`qvm-firewall` -- Qubes firewall configuration +======================================================= -:Date: 2012-04-10 - -SYNOPSIS +Synopsis ======== | qvm-firewall [-n] [action] [rule spec] @@ -17,29 +13,49 @@ Rule specification can be one of: 2. address|hostname[/netmask] tcp|udp service_name 3. address|hostname[/netmask] any -OPTIONS +Options ======= --h, --help - Show this help message and exit --l, --list - List firewall settings (default action) --a, --add - Add rule --d, --del - Remove rule (given by number or by rule spec) --P SET_POLICY, --policy=SET_POLICY - Set firewall policy (allow/deny) --i SET_ICMP, --icmp=SET_ICMP - Set ICMP access (allow/deny) --D SET_DNS, --dns=SET_DNS - Set DNS access (allow/deny) --Y SET_YUM_PROXY, --yum-proxy=SET_YUM_PROXY - Set access to Qubes yum proxy (allow/deny). - *Note:* if set to "deny", access will be rejected even if policy set to "allow" --n, --numeric - Display port numbers instead of services (makes sense only with --list) -AUTHORS +.. option:: --help, -h + + Show this help message and exit + +.. option:: --list, -l + + List firewall settings (default action) + +.. option:: --add, -a + + Add rule + +.. option:: --del, -d + + Remove rule (given by number or by rule spec) + +.. option:: --policy=SET_POLICY, -P SET_POLICY + + Set firewall policy (allow/deny) + +.. option:: --icmp=SET_ICMP, -i SET_ICMP + + Set ICMP access (allow/deny) + +.. option:: --dns=SET_DNS, -D SET_DNS + + Set DNS access (allow/deny) + +.. option:: --yum-proxy=SET_YUM_PROXY, -Y SET_YUM_PROXY + + Set access to Qubes yum proxy (allow/deny). + + .. note:: + if set to "deny", access will be rejected even if policy set to "allow" + +.. option:: --numeric, -n + + Display port numbers instead of services (makes sense only with :option:`--list`) + +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-grow-private.rst b/doc/qvm-tools/qvm-grow-private.rst index 9ac1c66e..33cb381c 100644 --- a/doc/qvm-tools/qvm-grow-private.rst +++ b/doc/qvm-tools/qvm-grow-private.rst @@ -1,23 +1,21 @@ -================ -qvm-grow-private -================ +.. program:: qvm-grow-private -NAME -==== -qvm-grow-private - increase private storage capacity of a specified VM +================================================================================== +:program:`qvm-grow-private` -- Increase private storage capacity of a specified VM +================================================================================== -:Date: 2012-04-10 - -SYNOPSIS +Synopsis ======== | qvm-grow-private -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-kill.rst b/doc/qvm-tools/qvm-kill.rst index 9c14f134..8bdf74a9 100644 --- a/doc/qvm-tools/qvm-kill.rst +++ b/doc/qvm-tools/qvm-kill.rst @@ -1,24 +1,22 @@ -======== -qvm-kill -======== +.. program:: qvm-kill -NAME -==== -qvm-kill - kills the specified VM +============================================ +:program:`qvm-kill` -- Kill the specified VM +============================================ -:Date: 2012-04-10 - -SYNOPSIS +Synopsis ======== | qvm-kill [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-ls.rst b/doc/qvm-tools/qvm-ls.rst index 2abbd9bc..c8f334aa 100644 --- a/doc/qvm-tools/qvm-ls.rst +++ b/doc/qvm-tools/qvm-ls.rst @@ -1,39 +1,53 @@ -====== -qvm-ls -====== +.. program:: qvm-ls -NAME -==== -qvm-ls - list VMs and various information about their state +================================================================ +:program:`qvm-ls` -- List VMs and various information about them +================================================================ -:Date: 2012-04-03 - -SYNOPSIS +Synopsis ======== | qvm-ls [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show help message and exit --n, --network + +.. option:: --network, -n + Show network addresses assigned to VMs --c, --cpu + +.. option:: --cpu, -c + Show CPU load --m, --mem + +.. option:: --mem, -m + Show memory usage --d, --disk + +.. option:: --disk, -d + Show VM disk utilization statistics --i, --ids + +.. option:: --ids, -i + Show Qubes and Xen id --k, --kernel + +.. option:: --kernel, -k + Show VM kernel options --b, --last-backup + +.. option:: --last-backup, -b + Show date of last VM backup ---raw-list + +.. option:: --raw-list + List only VM names one per line -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-pci.rst b/doc/qvm-tools/qvm-pci.rst index c2e667f8..72a9f4be 100644 --- a/doc/qvm-tools/qvm-pci.rst +++ b/doc/qvm-tools/qvm-pci.rst @@ -1,32 +1,35 @@ -======= -qvm-pci -======= +.. program:: qvm-pci -NAME -==== -qvm-pci - list/set VM PCI devices +============================================= +:program:`qvm-pci` -- List/set VM PCI devices +============================================= - -:Date: 2012-04-11 - -SYNOPSIS +Synopsis ======== | qvm-pci -l [options] | qvm-pci -a [options] | qvm-pci -d [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit --l, --list + +.. option:: --list, -l + List VM PCI devices --a, --add + +.. option:: --add, -a + Add a PCI device to specified VM --d, --delete + +.. option:: --delete, -d + Remove a PCI device from specified VM -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-prefs.rst b/doc/qvm-tools/qvm-prefs.rst index 197ef6eb..2aa5a548 100644 --- a/doc/qvm-tools/qvm-prefs.rst +++ b/doc/qvm-tools/qvm-prefs.rst @@ -1,32 +1,36 @@ -========= -qvm-prefs -========= +.. program:: qvm-prefs -NAME -==== -qvm-prefs - list/set various per-VM properties +========================================================== +:program:`qvm-prefs` -- List/set various per-VM properties +========================================================== -:Date: 2012-04-11 - -SYNOPSIS +Synopsis ======== | qvm-prefs -l [options] | qvm-prefs -g [options] | qvm-prefs -s [options] [...] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit --l, --list + +.. option:: --list, -l + List properties of a specified VM --g, --get + +.. option:: --get, -g + Get a single property of a specified VM --s, --set + +.. option:: --set, -s + Set properties of a specified VM -PROPERTIES +Properties ========== include_in_backups @@ -141,7 +145,7 @@ timezone Set emulated HVM clock timezone. Use ``localtime`` (the default) to use the same time as dom0 have. Note that HVM will get only clock value, not the timezone itself, so if you use ``localtime`` setting, OS inside of HVM should also be configured to treat hardware clock as local time (and have proper timezone set). -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-remove.rst b/doc/qvm-tools/qvm-remove.rst index b3a71aae..0db927e3 100644 --- a/doc/qvm-tools/qvm-remove.rst +++ b/doc/qvm-tools/qvm-remove.rst @@ -1,29 +1,33 @@ -========== -qvm-remove -========== +.. program:: qvm-remove -NAME -==== -qvm-remove - remove a VM +==================================== +:program:`qvm-remove` -- Remove a VM +==================================== -:Date: 2012-04-11 - -SYNOPSIS +Synopsis ======== | qvm-remove [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit --q, --quiet + +.. option:: --quiet, -q + Be quiet ---just-db + +.. option:: --just-db + Remove only from the Qubes Xen DB, do not remove any files ---force-root + +.. option:: --force-root + Force to run, even with root privileges -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-revert-template-changes.rst b/doc/qvm-tools/qvm-revert-template-changes.rst index f250cda2..5de5192c 100644 --- a/doc/qvm-tools/qvm-revert-template-changes.rst +++ b/doc/qvm-tools/qvm-revert-template-changes.rst @@ -1,25 +1,25 @@ -=========================== -qvm-revert-template-changes -=========================== +.. program:: qvm-revert-template-changes -NAME -==== -qvm-revert-template-changes +====================================================================== +:program:`qvm-revert-template-changes` -- Revert changes to a template +====================================================================== -:Date: 2012-04-11 - -SYNOPSIS +Synopsis ======== | qvm-revert-template-changes [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit ---force + +.. option:: --force + Do not prompt for comfirmation -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-run.rst b/doc/qvm-tools/qvm-run.rst index 15eb6779..1a7830d1 100644 --- a/doc/qvm-tools/qvm-run.rst +++ b/doc/qvm-tools/qvm-run.rst @@ -1,49 +1,79 @@ -======= -qvm-run -======= +.. program:: qvm-run -NAME -==== -qvm-run - run a command on a specified VM +===================================================== +:program:`qvm-run` -- Run a command on a specified VM +===================================================== -:Date: 2012-04-11 - -SYNOPSIS +Synopsis ======== | qvm-run [options] [] [] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit --q, --quiet + +.. option:: --quiet, -q + Be quiet --a, --auto + +.. option:: --auto, -a + Auto start the VM if not running --u USER, --user=USER + +.. option:: --user=USER, -u USER + Run command in a VM as a specified user ---tray + +.. option:: --tray + Use tray notifications instead of stdout ---all - Run command on all currently running VMs (or all paused, in case of --unpause) ---exclude=EXCLUDE_LIST - When --all is used: exclude this VM name (might be repeated) ---wait + +.. option:: --all + + Run command on all currently running VMs (or all paused, in case of :option:`--unpause`) + +.. option:: --exclude=EXCLUDE_LIST + + When :option:`--all` is used: exclude this VM name (might be repeated) + +.. option:: --wait + Wait for the VM(s) to shutdown ---shutdown - (deprecated) Do 'xl shutdown' for the VM(s) (can be combined this with --all and --wait) ---pause - Do 'xl pause' for the VM(s) (can be combined this with --all and --wait) ---unpause - Do 'xl unpause' for the VM(s) (can be combined this with --all and --wait) --p, --pass-io + +.. option:: --shutdown + + Do 'xl shutdown' for the VM(s) (can be combined with :option:`--all` and + :option:`--wait`) + + .. deprecated:: R2 + Use :manpage:`qvm-shutdown(1)` instead. + +.. option:: --pause + + Do 'xl pause' for the VM(s) (can be combined with :option:`--all` and + :option:`--wait`) + +.. option:: --unpause + + Do 'xl unpause' for the VM(s) (can be combined with :option:`--all` and + :option:`--wait`) + +.. option:: --pass-io, -p + Pass stdin/stdout/stderr from remote program ---localcmd=LOCALCMD - With --pass-io, pass stdin/stdout/stderr to the given program ---force + +.. option:: --localcmd=LOCALCMD + + With :option:`--pass-io`, pass stdin/stdout/stderr to the given program + +.. option:: --force + Force operation, even if may damage other VMs (eg. shutdown of NetVM) -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-service.rst b/doc/qvm-tools/qvm-service.rst index 3a552b07..82377156 100644 --- a/doc/qvm-tools/qvm-service.rst +++ b/doc/qvm-tools/qvm-service.rst @@ -1,33 +1,44 @@ -=========== -qvm-service -=========== +.. program:: qvm-service + +======================================================================== +:program:`qvm-service` -- Manage (Qubes-specific) services started in VM +======================================================================== NAME ==== -qvm-service - manage (Qubes-specific) services started in VM +qvm-service - :Date: 2012-05-30 -SYNOPSIS +Synopsis ======== | qvm-service [-l] | qvm-service [-e|-d|-D] -OPTIONS +Options ======= --h, --help +.. option:: --help, -h + Show this help message and exit --l, --list + +.. option:: --list, -l + List services (default action) --e, --enable + +.. option:: --enable, -e + Enable service --d, --disable + +.. option:: --disable, -d + Disable service --D, --default + +.. option:: --default, -D + Reset service to its default state (remove from the list). Default state means "lets VM choose" and can depend on VM type (NetVM, AppVM etc). -SUPPORTED SERVICES +Supported services ================== This list can be incomplete as VM can implement any additional service without knowlege of qubes-core code. @@ -109,9 +120,11 @@ updates-proxy-setup Setup yum at startup to use qubes-yum-proxy service. - *Note:* this service is automatically enabled when you allow VM to access - yum proxy (in firewall settings) and disabled when you deny access to yum - proxy. + .. note:: + + this service is automatically enabled when you allow VM to access yum + proxy (in firewall settings) and disabled when you deny access to yum + proxy. disable-default-route Default: disabled @@ -128,7 +141,7 @@ disable-dns-server The functionality is implemented in /usr/lib/qubes/setup-ip. -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-shutdown.rst b/doc/qvm-tools/qvm-shutdown.rst index d38fc4ea..4831c90a 100644 --- a/doc/qvm-tools/qvm-shutdown.rst +++ b/doc/qvm-tools/qvm-shutdown.rst @@ -1,33 +1,41 @@ -============ -qvm-shutdown -============ +.. program:: qvm-shutdown -NAME -==== -qvm-shutdown +==================================================== +:program:`qvm-shutdown` -- Gracefully shut down a VM +==================================================== -:Date: 2012-04-11 - -SYNOPSIS +Synopsis ======== | qvm-shutdown [options] -OPTIONS +Options ======= --h, --help - Show this help message and exit --q, --quiet - Be quiet ---force - Force operation, even if may damage other VMs (eg. shutdown of NetVM) ---wait - Wait for the VM(s) to shutdown ---all - Shutdown all running VMs ---exclude=EXCLUDE_LIST - When --all is used: exclude this VM name (might be repeated) -AUTHORS +.. option:: --help, -h + + Show this help message and exit + +.. option:: --quiet, -q + + Be quiet + +.. option:: --force + + Force operation, even if may damage other VMs (eg. shutdown of NetVM) + +.. option:: --wait + + Wait for the VM(s) to shutdown + +.. option:: --all + + Shutdown all running VMs + +.. option:: --exclude=EXCLUDE_LIST + + When :option:`--all` is used: exclude this VM name (might be repeated) + +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-start.rst b/doc/qvm-tools/qvm-start.rst index 3645b169..a880c0e1 100644 --- a/doc/qvm-tools/qvm-start.rst +++ b/doc/qvm-tools/qvm-start.rst @@ -1,33 +1,41 @@ -========= -qvm-start -========= +.. program:: qvm-start -NAME -==== -qvm-start - start a specified VM +============================================ +:program:`qvm-start` -- Start a specified VM +============================================ -:Date: 2012-04-11 - -SYNOPSIS +Synopsis ======== | qvm-start [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit --q, --quiet + +.. option:: --quiet, -q + Be quiet ---no-guid + +.. option:: --no-guid + Do not start the GUId (ignored) ---console + +.. option:: --console + Attach debugging console to the newly started VM ---dvm + +.. option:: --dvm + Do actions necessary when preparing DVM image ---custom-config=CUSTOM_CONFIG + +.. option:: --custom-config=CUSTOM_CONFIG + Use custom Xen config instead of Qubes-generated one -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/doc/qvm-tools/qvm-template-commit.rst b/doc/qvm-tools/qvm-template-commit.rst index bb2bf857..eb03bf6b 100644 --- a/doc/qvm-tools/qvm-template-commit.rst +++ b/doc/qvm-tools/qvm-template-commit.rst @@ -1,23 +1,21 @@ -=================== -qvm-template-commit -=================== +.. program:: qvm-template-commit -NAME -==== -qvm-template-commit +============================================================== +:program:`qvm-template-commit` -- Commit changes to a template +============================================================== -:Date: 2012-04-11 - -SYNOPSIS +Synopsis ======== | qvm-template-commit [options] -OPTIONS +Options ======= --h, --help + +.. option:: --help, -h + Show this help message and exit -AUTHORS +Authors ======= | Joanna Rutkowska | Rafal Wojtczuk diff --git a/rpm_spec/core-dom0-doc.spec b/rpm_spec/core-dom0-doc.spec index c4d6068d..e6694f5b 100644 --- a/rpm_spec/core-dom0-doc.spec +++ b/rpm_spec/core-dom0-doc.spec @@ -44,7 +44,7 @@ Provides: qubes-doc-dom0 The Qubes docs for dom0 tools %build -make manpages +make man %install From 56092073e9a3f607d7fc6ef7ce04fdc00a49c102 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Fri, 21 Nov 2014 12:34:27 +0100 Subject: [PATCH 0014/1004] doc: add hyphens to module page titles Aim is to be more consistent with manpages. --- doc/qubes-dochelpers.rst | 4 ++-- doc/qubes-events.rst | 4 ++-- doc/qubes-ext.rst | 4 ++-- doc/qubes-log.rst | 4 ++-- doc/qubes-plugins.rst | 4 ++-- doc/qubes-vm/adminvm.rst | 4 ++-- doc/qubes-vm/appvm.rst | 4 ++-- doc/qubes-vm/dispvm.rst | 4 ++-- doc/qubes-vm/hvm.rst | 4 ++-- doc/qubes-vm/index.rst | 4 ++-- doc/qubes-vm/netvm.rst | 4 ++-- doc/qubes-vm/proxyvm.rst | 4 ++-- doc/qubes-vm/qubesvm.rst | 4 ++-- doc/qubes-vm/templatehvm.rst | 4 ++-- doc/qubes-vm/templatevm.rst | 4 ++-- doc/qubes.rst | 4 ++-- 16 files changed, 32 insertions(+), 32 deletions(-) diff --git a/doc/qubes-dochelpers.rst b/doc/qubes-dochelpers.rst index 6f1c9ae1..0ba92754 100644 --- a/doc/qubes-dochelpers.rst +++ b/doc/qubes-dochelpers.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.dochelpers` Helpers for Sphinx documentation -=========================================================== +:py:mod:`qubes.dochelpers` -- Helpers for Sphinx documentation +============================================================== .. automodule:: qubes.dochelpers :members: diff --git a/doc/qubes-events.rst b/doc/qubes-events.rst index 941fe784..636042ef 100644 --- a/doc/qubes-events.rst +++ b/doc/qubes-events.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.events` Qubes events -=================================== +:py:mod:`qubes.events` -- Qubes events +====================================== .. automodule:: qubes.events :members: diff --git a/doc/qubes-ext.rst b/doc/qubes-ext.rst index ff71fe4e..8d59d9eb 100644 --- a/doc/qubes-ext.rst +++ b/doc/qubes-ext.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.ext` Qubes extensions -======================================== +:py:mod:`qubes.ext` -- Qubes extensions +======================================= .. automodule:: qubes.ext diff --git a/doc/qubes-log.rst b/doc/qubes-log.rst index f6fa3e7e..3d12a621 100644 --- a/doc/qubes-log.rst +++ b/doc/qubes-log.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.log` Logging routines -==================================== +:py:mod:`qubes.log` -- Logging routines +======================================= .. automodule:: qubes.log :members: diff --git a/doc/qubes-plugins.rst b/doc/qubes-plugins.rst index fa1630d2..48323d8c 100644 --- a/doc/qubes-plugins.rst +++ b/doc/qubes-plugins.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.plugins` Plugin helpers -====================================== +:py:mod:`qubes.plugins` -- Plugin helpers +========================================= .. automodule:: qubes.plugins :members: diff --git a/doc/qubes-vm/adminvm.rst b/doc/qubes-vm/adminvm.rst index 8f3edf7b..7f5a7b77 100644 --- a/doc/qubes-vm/adminvm.rst +++ b/doc/qubes-vm/adminvm.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.vm.adminvm` Dom0 -=============================== +:py:mod:`qubes.vm.adminvm` -- Dom0 +================================== .. automodule:: qubes.vm.adminvm :members: diff --git a/doc/qubes-vm/appvm.rst b/doc/qubes-vm/appvm.rst index 2f95be5b..5ef55f31 100644 --- a/doc/qubes-vm/appvm.rst +++ b/doc/qubes-vm/appvm.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.vm.appvm` Application VM -======================================= +:py:mod:`qubes.vm.appvm` -- Application VM +========================================== .. automodule:: qubes.vm.appvm :members: diff --git a/doc/qubes-vm/dispvm.rst b/doc/qubes-vm/dispvm.rst index f307e16f..f8ab796e 100644 --- a/doc/qubes-vm/dispvm.rst +++ b/doc/qubes-vm/dispvm.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.vm.dispvm` Disposable VM -======================================= +:py:mod:`qubes.vm.dispvm` -- Disposable VM +========================================== .. automodule:: qubes.vm.dispvm :members: diff --git a/doc/qubes-vm/hvm.rst b/doc/qubes-vm/hvm.rst index 477c3fc8..3aaff0d3 100644 --- a/doc/qubes-vm/hvm.rst +++ b/doc/qubes-vm/hvm.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.vm.hvm` HVM -========================== +:py:mod:`qubes.vm.hvm` -- HVM +============================= .. automodule:: qubes.vm.hvm :members: diff --git a/doc/qubes-vm/index.rst b/doc/qubes-vm/index.rst index 2d6733f3..f92ddaa5 100644 --- a/doc/qubes-vm/index.rst +++ b/doc/qubes-vm/index.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.vm` Different Virtual Machine types -================================================== +:py:mod:`qubes.vm` -- Different Virtual Machine types +===================================================== .. automodule:: qubes.vm diff --git a/doc/qubes-vm/netvm.rst b/doc/qubes-vm/netvm.rst index 8ca90d4d..74255edc 100644 --- a/doc/qubes-vm/netvm.rst +++ b/doc/qubes-vm/netvm.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.vm.netvm` Network interface VM -============================================= +:py:mod:`qubes.vm.netvm` -- Network interface VM +================================================ .. automodule:: qubes.vm.netvm :members: diff --git a/doc/qubes-vm/proxyvm.rst b/doc/qubes-vm/proxyvm.rst index 665f0ab9..592c94b8 100644 --- a/doc/qubes-vm/proxyvm.rst +++ b/doc/qubes-vm/proxyvm.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.vm.proxyvm` Proxy (firewall/VPN) VM -================================================== +:py:mod:`qubes.vm.proxyvm` -- Proxy (firewall/VPN) VM +===================================================== .. automodule:: qubes.vm.proxyvm :members: diff --git a/doc/qubes-vm/qubesvm.rst b/doc/qubes-vm/qubesvm.rst index f5119ea6..90f1b71a 100644 --- a/doc/qubes-vm/qubesvm.rst +++ b/doc/qubes-vm/qubesvm.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.vm.qubesvm` Shared functionality -=============================================== +:py:mod:`qubes.vm.qubesvm` -- Shared functionality +================================================== .. automodule:: qubes.vm.qubesvm :members: diff --git a/doc/qubes-vm/templatehvm.rst b/doc/qubes-vm/templatehvm.rst index 4f668220..81c1f2c7 100644 --- a/doc/qubes-vm/templatehvm.rst +++ b/doc/qubes-vm/templatehvm.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.vm.templatehvm` Template for HVM -=============================================== +:py:mod:`qubes.vm.templatehvm` -- Template for HVM +================================================== .. automodule:: qubes.vm.templatehvm :members: diff --git a/doc/qubes-vm/templatevm.rst b/doc/qubes-vm/templatevm.rst index 5842ada8..a7ffa66b 100644 --- a/doc/qubes-vm/templatevm.rst +++ b/doc/qubes-vm/templatevm.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes.vm.templatevm` Template for AppVM -================================================ +:py:mod:`qubes.vm.templatevm` -- Template for AppVM +=================================================== .. automodule:: qubes.vm.templatevm :members: diff --git a/doc/qubes.rst b/doc/qubes.rst index e2ae68b3..b4616ddd 100644 --- a/doc/qubes.rst +++ b/doc/qubes.rst @@ -1,5 +1,5 @@ -:py:mod:`qubes` Common concepts -=============================== +:py:mod:`qubes` -- Common concepts +================================== .. automodule:: qubes :members: From a4e1bd98ed1cf8c1149095e6b8e5596447a42b07 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Fri, 21 Nov 2014 14:41:00 +0100 Subject: [PATCH 0015/1004] doc: optional sandbox building --- doc/.gitignore | 2 +- doc/conf.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/.gitignore b/doc/.gitignore index 10d00b57..512245fb 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -1 +1 @@ -*.gz +sandbox.rst diff --git a/doc/conf.py b/doc/conf.py index 1a34519e..f000ffcf 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -277,6 +277,10 @@ man_pages = [ u'Display system-wide Qubes settings', _man_pages_author, 1), ] +if os.path.exists('sandbox.rst'): + man_pages.append(('sandbox', 'sandbox', + u'Sandbox manpage', 'Sandbox Author', 1)) + # If true, show URL addresses after external links. #man_show_urls = False From 6c68bd062e648252bc7c8d431636e63c80757f63 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Fri, 21 Nov 2014 14:47:40 +0100 Subject: [PATCH 0016/1004] qubes.dochelpers: Version check for manual pages --- qubes/dochelpers.py | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/qubes/dochelpers.py b/qubes/dochelpers.py index 95a333e1..5d0700bd 100644 --- a/qubes/dochelpers.py +++ b/qubes/dochelpers.py @@ -14,7 +14,11 @@ import sys import urllib2 import docutils +import docutils.nodes +import docutils.parsers.rst import docutils.parsers.rst.roles +import docutils.statemachine +import sphinx.locale def fetch_ticket_info(uri): '''Fetch info about particular trac ticket given @@ -69,8 +73,56 @@ def ticket(name, rawtext, text, lineno, inliner, options={}, content=[]): return [node], [] +class versioncheck(docutils.nodes.warning): pass + +def visit(self, node): + self.visit_admonition(node, 'version') + +def depart(self, node): + self.depart_admonition(node) + +sphinx.locale.admonitionlabels['version'] = 'Version mismatch' + + +class VersionCheck(docutils.parsers.rst.Directive): + '''Directive versioncheck + + Check if current version (from ``conf.py``) equals version specified as + argument. If not, generate warning.''' + + has_content = True + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = True + option_spec = {} + + def run(self): + current = self.state.document.settings.env.app.config.version + version = self.arguments[0] + + if current == version: + return [] + + text = ' '.join('''This manual page was written for version **{}**, but + current version at the time when this page was generated is **{}**. + This may or may not mean that page is outdated or has + inconsistencies.'''.format(version, current).split()) + + node = versioncheck(text) + node['classes'] = ['admonition', 'warning'] + + self.state.nested_parse(docutils.statemachine.StringList([text]), + self.content_offset, node) + return [node] + + def setup(app): app.add_role('ticket', ticket) app.add_config_value('ticket_base_uri', 'https://wiki.qubes-os.org/ticket/', 'env') + app.add_node(versioncheck, + html=(visit, depart), + man=(visit, depart)) + app.add_directive('versioncheck', VersionCheck) + # vim: ts=4 sw=4 et From 6146c8e46687d77f804d13a10bbc285ea6d30595 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Fri, 21 Nov 2014 16:51:59 +0100 Subject: [PATCH 0017/1004] QubesVmLabel: XML parsing --- qubes/__init__.py | 17 +++++++++++++++++ tests/init.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tests/init.py diff --git a/qubes/__init__.py b/qubes/__init__.py index adf0deec..74793219 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -8,6 +8,8 @@ __author__ = 'Invisible Things Lab' __license__ = 'GPLv2 or later' __version__ = 'R3' +import ast + import qubes._pluginloader class QubesException(Exception): @@ -168,6 +170,21 @@ class QubesVmLabel(object): #: freedesktop icon name, suitable for use in :py:meth:`PyQt4.QtGui.QIcon.fromTheme` self.icon = '{}-{}'.format(('dispvm' if dispvm else 'appvm'), name) + @classmethod + def fromxml(cls, xml): + '''Create label definition from XML node + + :param :py:class:`lxml.etree._Element` xml: XML node reference + :rtype: :py:class:`qubes.QubesVmLabel` + ''' + + index = int(xml.get('id').split('-', 1)[1]) + color = xml.get('color') + name = xml.text + dispvm = ast.literal_eval(xml.get('dispvm', 'False')) + + return cls(index, color, name, dispvm) + def __repr__(self): return '{}({!r}, {!r}, {!r}, dispvm={!r})'.format( self.__class__.__name__, diff --git a/tests/init.py b/tests/init.py new file mode 100644 index 00000000..c91e65c4 --- /dev/null +++ b/tests/init.py @@ -0,0 +1,43 @@ +#!/usr/bin/python2 -O + +import sys +import unittest + +import lxml.etree + +sys.path.insert(0, '../') +import qubes + +class TC_QubesVmLabel(unittest.TestCase): + def test_000_appvm(self): + xml = lxml.etree.XML(''' + + + + + + ''') + + node = xml.xpath('//label')[0] + label = qubes.QubesVmLabel.fromxml(node) + + self.assertEqual(label.index, 1) + self.assertEqual(label.color, '#cc0000') + self.assertEqual(label.name, 'red') + self.assertEqual(label.dispvm, False) + self.assertEqual(label.icon, 'appvm-red') + + def test_001_dispvm(self): + xml = lxml.etree.XML(''' + + + + + + ''') + + node = xml.xpath('//label')[0] + label = qubes.QubesVmLabel.fromxml(node) + + self.assertEqual(label.dispvm, True) + self.assertEqual(label.icon, 'dispvm-red') From 2835238a875bcf03bff999c5cf64c6dc3b3e50e3 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Fri, 28 Nov 2014 18:37:17 +0100 Subject: [PATCH 0018/1004] doc: manpage formatting --- doc/qubes-tools/qubes-guid.rst | 2 +- doc/qubes-tools/qubes-prefs.rst | 2 +- doc/qvm-tools/qvm-add-appvm.rst | 2 +- doc/qvm-tools/qvm-add-template.rst | 2 +- doc/qvm-tools/qvm-backup-restore.rst | 2 +- doc/qvm-tools/qvm-backup.rst | 2 +- doc/qvm-tools/qvm-block.rst | 8 +- doc/qvm-tools/qvm-clone-template.rst | 2 +- doc/qvm-tools/qvm-clone.rst | 2 +- doc/qvm-tools/qvm-create-default-dvm.rst | 2 +- doc/qvm-tools/qvm-create.rst | 2 +- doc/qvm-tools/qvm-firewall.rst | 8 +- doc/qvm-tools/qvm-grow-private.rst | 2 +- doc/qvm-tools/qvm-kill.rst | 2 +- doc/qvm-tools/qvm-ls.rst | 2 +- doc/qvm-tools/qvm-pci.rst | 6 +- doc/qvm-tools/qvm-prefs.rst | 114 ++++++++++++++---- doc/qvm-tools/qvm-remove.rst | 2 +- doc/qvm-tools/qvm-revert-template-changes.rst | 2 +- doc/qvm-tools/qvm-run.rst | 2 +- doc/qvm-tools/qvm-service.rst | 40 +++--- doc/qvm-tools/qvm-shutdown.rst | 2 +- doc/qvm-tools/qvm-start.rst | 2 +- doc/qvm-tools/qvm-template-commit.rst | 2 +- 24 files changed, 144 insertions(+), 70 deletions(-) diff --git a/doc/qubes-tools/qubes-guid.rst b/doc/qubes-tools/qubes-guid.rst index 61c6e4d8..598f354b 100644 --- a/doc/qubes-tools/qubes-guid.rst +++ b/doc/qubes-tools/qubes-guid.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qubes-guid -d domain_id [-c color] [-l label_index] [-i icon name, no suffix] [-v] [-q] +:command:`qubes-guid` -d *domain_id* [-c *color*] [-l *label_index*] [-i *icon name*, no suffix] [-v] [-q] Options ======= diff --git a/doc/qubes-tools/qubes-prefs.rst b/doc/qubes-tools/qubes-prefs.rst index ca0a4eb3..d8b89754 100644 --- a/doc/qubes-tools/qubes-prefs.rst +++ b/doc/qubes-tools/qubes-prefs.rst @@ -13,7 +13,7 @@ Synopsis ======== -| qubes-prefs +:command:`qubes-prefs` Authors ======= diff --git a/doc/qvm-tools/qvm-add-appvm.rst b/doc/qvm-tools/qvm-add-appvm.rst index 3ada1c79..65f54057 100644 --- a/doc/qvm-tools/qvm-add-appvm.rst +++ b/doc/qvm-tools/qvm-add-appvm.rst @@ -9,7 +9,7 @@ Synopsis ======== -| qvm-add-appvm [options] +:command:`qvm-add-appvm` [*options*] <*appvm-name*> <*vm-template-name*> Options ======= diff --git a/doc/qvm-tools/qvm-add-template.rst b/doc/qvm-tools/qvm-add-template.rst index 1bf889f0..0acc13b5 100644 --- a/doc/qvm-tools/qvm-add-template.rst +++ b/doc/qvm-tools/qvm-add-template.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-add-template [options] +:command:`qvm-add-template` [*options*] <*vm-template-name*> Options ======= diff --git a/doc/qvm-tools/qvm-backup-restore.rst b/doc/qvm-tools/qvm-backup-restore.rst index c2f6db75..4bc5e02a 100644 --- a/doc/qvm-tools/qvm-backup-restore.rst +++ b/doc/qvm-tools/qvm-backup-restore.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-backup-restore [options] +:command:`qvm-backup-restore` [*options*] <*backup-dir*> Options ======= diff --git a/doc/qvm-tools/qvm-backup.rst b/doc/qvm-tools/qvm-backup.rst index a0987caa..aa15fa53 100644 --- a/doc/qvm-tools/qvm-backup.rst +++ b/doc/qvm-tools/qvm-backup.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-backup [options] +:command:`qvm-backup` [*options*] <*backup-dir-path*> Options ======= diff --git a/doc/qvm-tools/qvm-block.rst b/doc/qvm-tools/qvm-block.rst index d637709e..b7230b15 100644 --- a/doc/qvm-tools/qvm-block.rst +++ b/doc/qvm-tools/qvm-block.rst @@ -6,10 +6,10 @@ Synopsis ======== -| qvm-block -l [options] -| qvm-block -a [options] -| qvm-block -d [options] -| qvm-block -d [options] +| :command:`qvm-block` [*options*] -l +| :command:`qvm-block` [*options*] -a <*device*> <*vm-name*> +| :command:`qvm-block` [*options*] -d <*device*> +| :command:`qvm-block` [*options*] -d <*vm-name*> Options diff --git a/doc/qvm-tools/qvm-clone-template.rst b/doc/qvm-tools/qvm-clone-template.rst index 112181e4..b96dfcbc 100644 --- a/doc/qvm-tools/qvm-clone-template.rst +++ b/doc/qvm-tools/qvm-clone-template.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-clone-template [options] +:command:`qvm-clone-template` [*options*] <*src-template-name*> <*new-template-name*> Options ======= diff --git a/doc/qvm-tools/qvm-clone.rst b/doc/qvm-tools/qvm-clone.rst index d75682af..1dd61554 100644 --- a/doc/qvm-tools/qvm-clone.rst +++ b/doc/qvm-tools/qvm-clone.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-clone [options] +:command:`qvm-clone` [*options*] <*src-name*> <*new-name*> Options ======= diff --git a/doc/qvm-tools/qvm-create-default-dvm.rst b/doc/qvm-tools/qvm-create-default-dvm.rst index 046fb768..3b716074 100644 --- a/doc/qvm-tools/qvm-create-default-dvm.rst +++ b/doc/qvm-tools/qvm-create-default-dvm.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-create-default-dvm templatename|--default-template|--used-template [script-name|--default-script] +| :command:`qvm-create-default-dvm` templatename|--default-template|--used-template [script-name|--default-script] OPTIONS ======= diff --git a/doc/qvm-tools/qvm-create.rst b/doc/qvm-tools/qvm-create.rst index 8c54428a..342302c3 100644 --- a/doc/qvm-tools/qvm-create.rst +++ b/doc/qvm-tools/qvm-create.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-create [options] +:command:`qvm-create` [*options*] <*vm-name*> Options ======= diff --git a/doc/qvm-tools/qvm-firewall.rst b/doc/qvm-tools/qvm-firewall.rst index 53837e90..840c78da 100644 --- a/doc/qvm-tools/qvm-firewall.rst +++ b/doc/qvm-tools/qvm-firewall.rst @@ -6,12 +6,12 @@ Synopsis ======== -| qvm-firewall [-n] [action] [rule spec] +:command:`qvm-firewall` [-n] <*vm-name*> [*action*] [*rule spec*] Rule specification can be one of: - 1. address|hostname[/netmask] tcp|udp port[-port] - 2. address|hostname[/netmask] tcp|udp service_name - 3. address|hostname[/netmask] any + 1. *address*\ |\ *hostname*\ [/*netmask*] tcp|udp *port*\ [-*port*] + 2. *address*\ |\ *hostname*\ [/*netmask*] tcp|udp *service_name* + 3. *address*\ |\ *hostname*\ [/*netmask*] any Options ======= diff --git a/doc/qvm-tools/qvm-grow-private.rst b/doc/qvm-tools/qvm-grow-private.rst index 33cb381c..8e4a7663 100644 --- a/doc/qvm-tools/qvm-grow-private.rst +++ b/doc/qvm-tools/qvm-grow-private.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-grow-private +:command:`qvm-grow-private` <*vm-name*> <*size*> Options ======= diff --git a/doc/qvm-tools/qvm-kill.rst b/doc/qvm-tools/qvm-kill.rst index 8bdf74a9..6462c208 100644 --- a/doc/qvm-tools/qvm-kill.rst +++ b/doc/qvm-tools/qvm-kill.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-kill [options] +:command:`qvm-kill` [*options*] <*vm-name*> Options diff --git a/doc/qvm-tools/qvm-ls.rst b/doc/qvm-tools/qvm-ls.rst index c8f334aa..cc722cda 100644 --- a/doc/qvm-tools/qvm-ls.rst +++ b/doc/qvm-tools/qvm-ls.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-ls [options] +:command:`qvm-ls` [*options*] <*vm-name*> Options ======= diff --git a/doc/qvm-tools/qvm-pci.rst b/doc/qvm-tools/qvm-pci.rst index 72a9f4be..c72b7205 100644 --- a/doc/qvm-tools/qvm-pci.rst +++ b/doc/qvm-tools/qvm-pci.rst @@ -6,9 +6,9 @@ Synopsis ======== -| qvm-pci -l [options] -| qvm-pci -a [options] -| qvm-pci -d [options] +| :command:`qvm-pci` [*options*] -l <*vm-name*> +| :command:`qvm-pci` [*options*] -a <*vm-name*> <*device*> +| :command:`qvm-pci` [*options*] -d <*vm-name*> <*device*> Options ======= diff --git a/doc/qvm-tools/qvm-prefs.rst b/doc/qvm-tools/qvm-prefs.rst index 2aa5a548..8ca298d3 100644 --- a/doc/qvm-tools/qvm-prefs.rst +++ b/doc/qvm-tools/qvm-prefs.rst @@ -6,9 +6,9 @@ Synopsis ======== -| qvm-prefs -l [options] -| qvm-prefs -g [options] -| qvm-prefs -s [options] [...] +| :command:`qvm-prefs` [*options*] -l <*vm-name*> +| :command:`qvm-prefs` [*options*] -g <*vm-name*> <*property*> +| :command:`qvm-prefs` [*options*] -s <*vm-name*> <*property*> [*...*] Options @@ -36,10 +36,13 @@ Properties include_in_backups Accepted values: ``True``, ``False`` - Control whenever this VM will be included in backups by default (for now works only in qubes-manager). You can always manually select or deselect any VM for backup. + Control whenever this VM will be included in backups by default (for now + works only in qubes-manager). You can always manually select or deselect + any VM for backup. pcidevs - PCI devices assigned to the VM. Should be edited using qvm-pci tool. + PCI devices assigned to the VM. Should be edited using + :manpage:`qvm-pci(1)` tool. pci_strictreset Accepted values: ``True``, ``False`` @@ -52,14 +55,24 @@ pci_strictreset is trusted one, or is running all the time. label - Accepted values: ``red``, ``orange``, ``yellow``, ``green``, ``gray``, ``blue``, ``purple``, ``black`` + Accepted values: ``red``, ``orange``, ``yellow``, ``green``, ``gray``, + ``blue``, ``purple``, ``black`` - Color of VM label (icon, appmenus, windows border). If VM is running, change will be applied at first VM restart. + Color of VM label (icon, appmenus, windows border). If VM is running, + change will be applied at first VM restart. netvm Accepted values: netvm name, ``default``, ``none`` - To which NetVM connect. Setting to ``default`` will follow system-global default NetVM (managed by qubes-prefs). Setting to ``none`` will disable networking in this VM. + To which NetVM connect. Setting to ``default`` will follow system-global + default NetVM (managed by qubes-prefs). Setting to ``none`` will disable + networking in this VM. + + .. note:: + + When setting to ``none``, firewall will be set to block all traffic - + it will be used by DispVM started from this VM. Setting back to some + NetVM will *NOT* restore previous firewall settings. dispvm_netvm Accepted values: netvm name, ``default``, ``none`` @@ -69,17 +82,27 @@ dispvm_netvm maxmem Accepted values: memory size in MB - Maximum memory size available for this VM. Dynamic memory management (aka qmemman) will not be able to balloon over this limit. For VMs with qmemman disabled, this will be overridden by *memory* property (at VM startup). + Maximum memory size available for this VM. Dynamic memory management (aka + qmemman) will not be able to balloon over this limit. For VMs with qmemman + disabled, this will be overridden by *memory* property (at VM startup). memory Accepted values: memory size in MB - Initial memory size for VM. This should be large enough to allow VM startup - before qmemman starts managing memory for this VM. For VM with qmemman disabled, this is static memory size. + Initial memory size for VM. This should be large enough to allow VM startup + - before qmemman starts managing memory for this VM. For VM with qmemman + disabled, this is static memory size. kernel Accepted values: kernel version, ``default``, ``none`` - Kernel version to use (only for PV VMs). Available kernel versions will be listed when no value given (there are in /var/lib/qubes/vm-kernels). Setting to ``default`` will follow system-global default kernel (managed via qubes-prefs). Setting to ``none`` will use "kernels" subdir in VM directory - this allows having VM-specific kernel; also this the only case when /lib/modules is writable from within VM. + Kernel version to use (only for PV VMs). Available kernel versions will be + listed when no value given (there are in + :file:`/var/lib/qubes/vm-kernels`). Setting to ``default`` will follow + system-global default kernel (managed via qubes-prefs). Setting to ``none`` + will use "kernels" subdir in VM directory - this allows having VM-specific + kernel; also this the only case when :file:`/lib/modules` is writable from + within VM. template Accepted values: TemplateVM name @@ -89,12 +112,17 @@ template vcpus Accepted values: no of CPUs - Number of CPU (cores) available to VM. Some VM types (eg DispVM) will not work properly with more than one CPU. + Number of CPU (cores) available to VM. Some VM types (eg DispVM) will not + work properly with more than one CPU. kernelopts Accepted values: string, ``default`` - VM kernel parameters (available only for PV VMs). This can be used to workaround some hardware specific problems (eg for NetVM). Setting to ``default`` will use some reasonable defaults (currently different for VMs with PCI devices and without). Some helpful options (for debugging purposes): ``earlyprintk=xen``, ``init=/bin/bash`` + VM kernel parameters (available only for PV VMs). This can be used to + workaround some hardware specific problems (eg for NetVM). Setting to + ``default`` will use some reasonable defaults (currently different for VMs + with PCI devices and without). Some helpful options (for debugging + purposes): ``earlyprintk=xen``, ``init=/bin/bash`` name Accepted values: alphanumerical name @@ -102,48 +130,86 @@ name Name of the VM. Can be only changed when VM isn't running. drive - Accepted values: [hd:\|cdrom:][backend-vm:]path + Accepted values: [hd:\|cdrom:][backend-vm:]\ *path* - Additional drive for the VM (available only for HVMs). This can be used to attach installation image. ``path`` can be file or physical device (eg. /dev/sr0). The same syntax can be used in qvm-start --drive - to attach drive only temporarily. + Additional drive for the VM (available only for HVMs). This can be used to + attach installation image. ``path`` can be file or physical device (eg. + :file:`/dev/sr0`). The same syntax can be used in :option:`qvm-start + --drive` - to attach drive only temporarily. mac Accepted values: MAC address, ``auto`` - Can be used to force specific of virtual ethernet card in the VM. Setting to ``auto`` will use automatic-generated MAC - based on VM id. Especially useful when some licencing depending on static MAC address. + Can be used to force specific of virtual ethernet card in the VM. Setting + to ``auto`` will use automatic-generated MAC - based on VM id. Especially + useful when some licencing depending on static MAC address. + For template-based HVM ``auto`` mode means to clone template MAC. default_user Accepted values: username - Default user used by qvm-run. Note that it make sense only on non-standard template, as the standard one always have "user" account. + Default user used by :manpage:`qvm-run(1)`. Note that it make sense only on + non-standard template, as the standard one always have "user" account. debug Accepted values: ``on``, ``off`` - Enables debug mode for VM. This can be used to turn on/off verbose logging in many qubes components at once (gui virtualization, VM kernel, some other services). - For template-based HVM, enabling debug mode also disables automatic reset root.img (actually volatile.img) before each VM startup, so changes made to root filesystem stays intact. To force reset root.img when debug mode enabled, either change something in the template (simple start+stop will do, even touch its root.img is enough), or remove VM's volatile.img (check the path with qvm-prefs). + Enables debug mode for VM. This can be used to turn on/off verbose logging + in many qubes components at once (gui virtualization, VM kernel, some other + services). + + For template-based HVM, enabling debug mode also disables automatic reset + :file:`root.img` (actually :file:`volatile.img`) before each VM startup, so + changes made to root filesystem stays intact. To force reset + :file:`root.img` when debug mode enabled, either change something in the + template (simple start+stop will do, even touch its root.img is enough), or + remove VM's :file:`volatile.img` (check the path with + :manpage:`qvm-prefs(1)`). qrexec_installed Accepted values: ``True``, ``False`` - This HVM have qrexec agent installed. When VM have qrexec agent installed, one can use qvm-run to start VM process, VM will benefit from Qubes RPC services (like file copy, or inter-vm clipboard). This option will be automatically turned on during Qubes Windows Tools installation, but if you install qrexec agent in some other OS, you need to turn this option on manually. + This HVM have qrexec agent installed. When VM have qrexec agent installed, + one can use qvm-run to start VM process, VM will benefit from Qubes RPC + services (like file copy, or inter-vm clipboard). This option will be + automatically turned on during Qubes Windows Tools installation, but if you + install qrexec agent in some other OS, you need to turn this option on + manually. guiagent_installed Accepted values: ``True``, ``False`` - This HVM have gui agent installed. This option disables full screen GUI virtualization and enables per-window seemless GUI mode. This option will be automatically turned on during Qubes Windows Tools installation, but if you install qubes gui agent in some other OS, you need to turn this option on manually. You can turn this option off to troubleshoot some early HVM OS boot problems (enter safe mode etc), but the option will be automatically enabled at first VM normal startup (and will take effect from the next startup). + This HVM have gui agent installed. This option disables full screen GUI + virtualization and enables per-window seemless GUI mode. This option will + be automatically turned on during Qubes Windows Tools installation, but if + you install qubes gui agent in some other OS, you need to turn this option + on manually. You can turn this option off to troubleshoot some early HVM OS + boot problems (enter safe mode etc), but the option will be automatically + enabled at first VM normal startup (and will take effect from the next + startup). - *Notice:* when Windows GUI agent is installed in the VM, SVGA device (used to full screen video) is disabled, so even if you disable this option, you will not get functional full desktop access (on normal VM startup). Use some other means for that (VNC, RDP or so). + .. note:: + + when Windows GUI agent is installed in the VM, SVGA device (used to + full screen video) is disabled, so even if you disable this option, you + will not get functional full desktop access (on normal VM startup). Use + some other means for that (VNC, RDP or so). autostart Accepted values: ``True``, ``False`` - Start the VM during system startup. The default netvm is autostarted regardless of this setting. + Start the VM during system startup. The default netvm is autostarted + regardless of this setting. timezone Accepted values: ``localtime``, time offset in seconds - Set emulated HVM clock timezone. Use ``localtime`` (the default) to use the same time as dom0 have. Note that HVM will get only clock value, not the timezone itself, so if you use ``localtime`` setting, OS inside of HVM should also be configured to treat hardware clock as local time (and have proper timezone set). + Set emulated HVM clock timezone. Use ``localtime`` (the default) to use the + same time as dom0 have. Note that HVM will get only clock value, not the + timezone itself, so if you use ``localtime`` setting, OS inside of HVM + should also be configured to treat hardware clock as local time (and have + proper timezone set). Authors ======= diff --git a/doc/qvm-tools/qvm-remove.rst b/doc/qvm-tools/qvm-remove.rst index 0db927e3..11ab319b 100644 --- a/doc/qvm-tools/qvm-remove.rst +++ b/doc/qvm-tools/qvm-remove.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-remove [options] +:command:`qvm-remove` [*options*] <*vm-name*> Options ======= diff --git a/doc/qvm-tools/qvm-revert-template-changes.rst b/doc/qvm-tools/qvm-revert-template-changes.rst index 5de5192c..dee5c377 100644 --- a/doc/qvm-tools/qvm-revert-template-changes.rst +++ b/doc/qvm-tools/qvm-revert-template-changes.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-revert-template-changes [options] +:command:`qvm-revert-template-changes` [*options*] <*template-name*> Options ======= diff --git a/doc/qvm-tools/qvm-run.rst b/doc/qvm-tools/qvm-run.rst index 1a7830d1..510debd3 100644 --- a/doc/qvm-tools/qvm-run.rst +++ b/doc/qvm-tools/qvm-run.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-run [options] [] [] +:command:`qvm-run` [*options*] [<*vm-name*>] [<*cmd*>] Options ======= diff --git a/doc/qvm-tools/qvm-service.rst b/doc/qvm-tools/qvm-service.rst index 82377156..e5afa7a5 100644 --- a/doc/qvm-tools/qvm-service.rst +++ b/doc/qvm-tools/qvm-service.rst @@ -4,16 +4,10 @@ :program:`qvm-service` -- Manage (Qubes-specific) services started in VM ======================================================================== -NAME -==== -qvm-service - - -:Date: 2012-05-30 - Synopsis ======== -| qvm-service [-l] -| qvm-service [-e|-d|-D] +| :command:`qvm-service` [-l] <*vmname*> +| :command:`qvm-service` [-e|-d|-D] <*vmname*> <*service*> Options ======= @@ -41,16 +35,20 @@ Options Supported services ================== -This list can be incomplete as VM can implement any additional service without knowlege of qubes-core code. +This list can be incomplete as VM can implement any additional service without +knowlege of qubes-core code. meminfo-writer Default: enabled everywhere excluding NetVM - This service reports VM memory usage to dom0, which effectively enables dynamic memory management for the VM. + This service reports VM memory usage to dom0, which effectively enables + dynamic memory management for the VM. - *Note:* this service is enforced to be set by dom0 code. If you try to - remove it (reset to defult state), will be recreated with the rule: enabled - if VM have no PCI devices assigned, otherwise disabled. + .. note:: + + This service is enforced to be set by dom0 code. If you try to + remove it (reset to defult state), will be recreated with the rule: enabled + if VM have no PCI devices assigned, otherwise disabled. qubes-dvm Default: disabled @@ -68,21 +66,31 @@ qubes-network Expose network for other VMs. This includes enabling network forwarding, MASQUERADE, DNS redirection and basic firewall. +qubes-network + Default: enabled only in NetVM and ProxyVM + + Expose network for other VMs. This includes enabling network forwarding, + MASQUERADE, DNS redirection and basic firewall. + qubes-netwatcher Default: enabled only in ProxyVM - Monitor IP change notification from NetVM. When received, reload qubes-firewall service (to force DNS resolution). + Monitor IP change notification from NetVM. When received, reload + qubes-firewall service (to force DNS resolution). + This service makes sense only with qubes-firewall enabled. qubes-update-check Default: enabled - Notify dom0 about updates available for this VM. This is shown in qubes-manager as 'update-pending' flag. + Notify dom0 about updates available for this VM. This is shown in + qubes-manager as 'update-pending' flag. cups Default: enabled only in AppVM - Enable CUPS service. The user can disable cups in VM which do not need printing to speed up booting. + Enable CUPS service. The user can disable cups in VM which do not need + printing to speed up booting. cron Default: disabled diff --git a/doc/qvm-tools/qvm-shutdown.rst b/doc/qvm-tools/qvm-shutdown.rst index 4831c90a..5ba25506 100644 --- a/doc/qvm-tools/qvm-shutdown.rst +++ b/doc/qvm-tools/qvm-shutdown.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-shutdown [options] +:command:`qvm-shutdown` [*options*] <*vm-name*> Options ======= diff --git a/doc/qvm-tools/qvm-start.rst b/doc/qvm-tools/qvm-start.rst index a880c0e1..ff68d957 100644 --- a/doc/qvm-tools/qvm-start.rst +++ b/doc/qvm-tools/qvm-start.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-start [options] +:command:`qvm-start` [*options*] <*vm-name*> Options ======= diff --git a/doc/qvm-tools/qvm-template-commit.rst b/doc/qvm-tools/qvm-template-commit.rst index eb03bf6b..38ab9400 100644 --- a/doc/qvm-tools/qvm-template-commit.rst +++ b/doc/qvm-tools/qvm-template-commit.rst @@ -6,7 +6,7 @@ Synopsis ======== -| qvm-template-commit [options] +:command:`qvm-template-commit` [*options*] <*vm-name*> Options ======= From b623a71d8705b96d637fe0496d35e586d4f173a4 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Fri, 5 Dec 2014 14:58:05 +0100 Subject: [PATCH 0019/1004] core3 move: QubesVmCollection This got split to qubes.Qubes and qubes.VMCollection. From now on, VMCollection is a stupid bag. Some parts went elsewhere. --- core/qubes.py | 625 ---------------------------------- qubes/__init__.py | 746 ++++++++++++++++++++++++++++++++++++++--- qubes/vm/__init__.py | 207 +++++++----- qubes/vm/qubesvm.py | 18 +- qubes/vm/templatevm.py | 5 + tests/init.py | 184 +++++++++- tests/vm.py | 23 +- 7 files changed, 1029 insertions(+), 779 deletions(-) diff --git a/core/qubes.py b/core/qubes.py index 574b4894..5d43d0e3 100755 --- a/core/qubes.py +++ b/core/qubes.py @@ -21,42 +21,6 @@ # # -from __future__ import absolute_import - -import atexit -import grp -import logging -import os -import os.path -import sys -import tempfile -import time -import warnings -import xml.parsers.expat - -import lxml.etree - -if os.name == 'posix': - import fcntl -elif os.name == 'nt': - import win32con - import win32file - import pywintypes -else: - raise RuntimeError, "Qubes works only on POSIX or WinNT systems" - -# Do not use XenAPI or create/read any VM files -# This is for testing only! -dry_run = False -#dry_run = True - -if not dry_run: - import libvirt - try: - import xen.lowlevel.xs - except ImportError: - pass - qubes_base_dir = "/var/lib/qubes" system_path = { @@ -134,595 +98,6 @@ def register_qubes_vm_class(vm_class): # other modules setattr(sys.modules[__name__], vm_class.__name__, vm_class) -class QubesVmCollection(dict): - """ - A collection of Qubes VMs indexed by Qubes id (qid) - """ - - def __init__(self, store_filename=None): - super(QubesVmCollection, self).__init__() - self.default_netvm_qid = None - self.default_fw_netvm_qid = None - self.default_template_qid = None - self.default_kernel = None - self.updatevm_qid = None - self.qubes_store_filename = store_filename - if not store_filename: - self.qubes_store_filename = system_path["qubes_store_filename"] - self.clockvm_qid = None - self.qubes_store_file = None - - self.log = logging.getLogger('qubes.qvmc.{:x}'.format(id(self))) - self.log.debug('instantiated store_filename={!r}'.format( - self.qubes_store_filename)) - - def __repr__(self): - return '<{} {!r}>'.format(self.__class__.__name__, list(sorted(self.keys()))) - - def clear(self): - self.log.debug('clear()') - super(QubesVmCollection, self).clear() - - def values(self): - for qid in self.keys(): - yield self[qid] - - def items(self): - for qid in self.keys(): - yield (qid, self[qid]) - - def __iter__(self): - for qid in sorted(super(QubesVmCollection, self).keys()): - yield qid - - keys = __iter__ - - def __setitem__(self, key, value): - self.log.debug('[{!r}] = {!r}'.format(key, value)) - if key not in self: - return super(QubesVmCollection, self).__setitem__(key, value) - else: - assert False, "Attempt to add VM with qid that already exists in the collection!" - - def add_new_vm(self, vm_type, **kwargs): - self.log.debug('add_new_vm(vm_type={}, **kwargs={!r})'.format( - vm_type, kwargs)) - if vm_type not in QubesVmClasses.keys(): - raise ValueError("Unknown VM type: %s" % vm_type) - - qid = self.get_new_unused_qid() - vm_cls = QubesVmClasses[vm_type] - if 'template' in kwargs: - if not vm_cls.is_template_compatible(kwargs['template']): - raise QubesException("Template not compatible with selected " - "VM type") - - vm = vm_cls(qid=qid, collection=self, **kwargs) - if not self.verify_new_vm(vm): - raise QubesException("Wrong VM description!") - self[vm.qid] = vm - - # make first created NetVM the default one - if self.default_fw_netvm_qid is None and vm.is_netvm(): - self.set_default_fw_netvm(vm) - - if self.default_netvm_qid is None and vm.is_proxyvm(): - self.set_default_netvm(vm) - - # make first created TemplateVM the default one - if self.default_template_qid is None and vm.is_template(): - self.set_default_template(vm) - - # make first created ProxyVM the UpdateVM - if self.updatevm_qid is None and vm.is_proxyvm(): - self.set_updatevm_vm(vm) - - # by default ClockVM is the first NetVM - if self.clockvm_qid is None and vm.is_netvm(): - self.set_clockvm_vm(vm) - - return vm - - def add_new_appvm(self, name, template, - dir_path = None, conf_file = None, - private_img = None, - label = None): - - warnings.warn("Call to deprecated function, use add_new_vm instead", - DeprecationWarning, stacklevel=2) - return self.add_new_vm("QubesAppVm", name=name, template=template, - dir_path=dir_path, conf_file=conf_file, - private_img=private_img, - netvm = self.get_default_netvm(), - kernel = self.get_default_kernel(), - uses_default_kernel = True, - label=label) - - def add_new_hvm(self, name, label = None): - - warnings.warn("Call to deprecated function, use add_new_vm instead", - DeprecationWarning, stacklevel=2) - return self.add_new_vm("QubesHVm", name=name, label=label) - - def add_new_disposablevm(self, name, template, dispid, - label = None, netvm = None): - - warnings.warn("Call to deprecated function, use add_new_vm instead", - DeprecationWarning, stacklevel=2) - return self.add_new_vm("QubesDisposableVm", name=name, template=template, - netvm = netvm, - label=label, dispid=dispid) - - def add_new_templatevm(self, name, - dir_path = None, conf_file = None, - root_img = None, private_img = None, - installed_by_rpm = True): - - warnings.warn("Call to deprecated function, use add_new_vm instead", - DeprecationWarning, stacklevel=2) - return self.add_new_vm("QubesTemplateVm", name=name, - dir_path=dir_path, conf_file=conf_file, - root_img=root_img, private_img=private_img, - installed_by_rpm=installed_by_rpm, - netvm = self.get_default_netvm(), - kernel = self.get_default_kernel(), - uses_default_kernel = True) - - def add_new_netvm(self, name, template, - dir_path = None, conf_file = None, - private_img = None, installed_by_rpm = False, - label = None): - - warnings.warn("Call to deprecated function, use add_new_vm instead", - DeprecationWarning, stacklevel=2) - return self.add_new_vm("QubesNetVm", name=name, template=template, - label=label, - private_img=private_img, installed_by_rpm=installed_by_rpm, - uses_default_kernel = True, - dir_path=dir_path, conf_file=conf_file) - - def add_new_proxyvm(self, name, template, - dir_path = None, conf_file = None, - private_img = None, installed_by_rpm = False, - label = None): - - warnings.warn("Call to deprecated function, use add_new_vm instead", - DeprecationWarning, stacklevel=2) - return self.add_new_vm("QubesProxyVm", name=name, template=template, - label=label, - private_img=private_img, installed_by_rpm=installed_by_rpm, - dir_path=dir_path, conf_file=conf_file, - uses_default_kernel = True, - netvm = self.get_default_fw_netvm()) - - def set_default_template(self, vm): - self.log.debug('set_default_template({!r})'.format(vm)) - if vm is None: - self.default_template_qid = None - else: - assert vm.is_template(), "VM {0} is not a TemplateVM!".format(vm.name) - self.default_template_qid = vm.qid - - def get_default_template(self): - if self.default_template_qid is None: - return None - else: - return self[self.default_template_qid] - - def set_default_netvm(self, vm): - self.log.debug('set_default_netvm({!r})'.format(vm)) - if vm is None: - self.default_netvm_qid = None - else: - assert vm.is_netvm(), "VM {0} does not provide network!".format(vm.name) - self.default_netvm_qid = vm.qid - - def get_default_netvm(self): - if self.default_netvm_qid is None: - return None - else: - return self[self.default_netvm_qid] - - def set_default_kernel(self, kernel): - self.log.debug('set_default_kernel({!r})'.format(kernel)) - assert os.path.exists( - os.path.join(system_path["qubes_kernels_base_dir"], kernel)), \ - "Kerel {0} not installed!".format(kernel) - self.default_kernel = kernel - - def get_default_kernel(self): - return self.default_kernel - - def set_default_fw_netvm(self, vm): - self.log.debug('set_default_fw_netvm({!r})'.format(vm)) - if vm is None: - self.default_fw_netvm_qid = None - else: - assert vm.is_netvm(), "VM {0} does not provide network!".format(vm.name) - self.default_fw_netvm_qid = vm.qid - - def get_default_fw_netvm(self): - if self.default_fw_netvm_qid is None: - return None - else: - return self[self.default_fw_netvm_qid] - - def set_updatevm_vm(self, vm): - self.log.debug('set_updatevm_vm({!r})'.format(vm)) - if vm is None: - self.updatevm_qid = None - else: - self.updatevm_qid = vm.qid - - def get_updatevm_vm(self): - if self.updatevm_qid is None: - return None - else: - return self[self.updatevm_qid] - - def set_clockvm_vm(self, vm): - self.log.debug('set_clockvm({!r})'.format(vm)) - if vm is None: - self.clockvm_qid = None - else: - self.clockvm_qid = vm.qid - - def get_clockvm_vm(self): - if self.clockvm_qid is None: - return None - else: - return self[self.clockvm_qid] - - def get_vm_by_name(self, name): - for vm in self.values(): - if (vm.name == name): - return vm - return None - - def get_qid_by_name(self, name): - vm = self.get_vm_by_name(name) - return vm.qid if vm is not None else None - - def get_vms_based_on(self, template_qid): - vms = set([vm for vm in self.values() - if (vm.template and vm.template.qid == template_qid)]) - return vms - - def get_vms_connected_to(self, netvm_qid): - new_vms = [ netvm_qid ] - dependend_vms_qid = [] - - # Dependency resolving only makes sense on NetVM (or derivative) - if not self[netvm_qid].is_netvm(): - return set([]) - - while len(new_vms) > 0: - cur_vm = new_vms.pop() - for vm in self[cur_vm].connected_vms.values(): - if vm.qid not in dependend_vms_qid: - dependend_vms_qid.append(vm.qid) - if vm.is_netvm(): - new_vms.append(vm.qid) - - vms = [vm for vm in self.values() if vm.qid in dependend_vms_qid] - return vms - - def verify_new_vm(self, new_vm): - - # Verify that qid is unique - for vm in self.values(): - if vm.qid == new_vm.qid: - print >> sys.stderr, "ERROR: The qid={0} is already used by VM '{1}'!".\ - format(vm.qid, vm.name) - return False - - # Verify that name is unique - for vm in self.values(): - if vm.name == new_vm.name: - print >> sys.stderr, \ - "ERROR: The name={0} is already used by other VM with qid='{1}'!".\ - format(vm.name, vm.qid) - return False - - return True - - def get_new_unused_qid(self): - used_ids = set([vm.qid for vm in self.values()]) - for id in range (1, qubes_max_qid): - if id not in used_ids: - return id - raise LookupError ("Cannot find unused qid!") - - def get_new_unused_netid(self): - used_ids = set([vm.netid for vm in self.values() if vm.is_netvm()]) - for id in range (1, qubes_max_netid): - if id not in used_ids: - return id - raise LookupError ("Cannot find unused netid!") - - - def check_if_storage_exists(self): - try: - f = open (self.qubes_store_filename, 'r') - except IOError: - return False - f.close() - return True - - def create_empty_storage(self): - self.log.debug('create_empty_storage()') - self.qubes_store_file = open (self.qubes_store_filename, 'w') - self.clear() - self.save() - - def lock_db_for_reading(self): - if self.qubes_store_file is not None: - raise QubesException("lock already taken") - # save() would rename the file over qubes.xml, _then_ release lock, - # so we need to ensure that the file for which we've got the lock is - # still the right file - self.log.debug('lock_db_for_reading()') - while True: - self.qubes_store_file = open (self.qubes_store_filename, 'r') - if os.name == 'posix': - fcntl.lockf (self.qubes_store_file, fcntl.LOCK_SH) - elif os.name == 'nt': - overlapped = pywintypes.OVERLAPPED() - win32file.LockFileEx(win32file._get_osfhandle(self.qubes_store_file.fileno()), - 0, 0, -0x10000, overlapped) - if os.fstat(self.qubes_store_file.fileno()) == os.stat( - self.qubes_store_filename): - break - self.qubes_store_file.close() - - def lock_db_for_writing(self): - if self.qubes_store_file is not None: - raise QubesException("lock already taken") - # save() would rename the file over qubes.xml, _then_ release lock, - # so we need to ensure that the file for which we've got the lock is - # still the right file - self.log.debug('lock_db_for_writing()') - while True: - self.qubes_store_file = open (self.qubes_store_filename, 'r+') - if os.name == 'posix': - fcntl.lockf (self.qubes_store_file, fcntl.LOCK_EX) - elif os.name == 'nt': - overlapped = pywintypes.OVERLAPPED() - win32file.LockFileEx(win32file._get_osfhandle(self.qubes_store_file.fileno()), - win32con.LOCKFILE_EXCLUSIVE_LOCK, 0, -0x10000, overlapped) - if os.fstat(self.qubes_store_file.fileno()) == os.stat( - self.qubes_store_filename): - break - self.qubes_store_file.close() - - def unlock_db(self): - # intentionally do not call explicit unlock to not unlock the file - # before all buffers are flushed - self.log.debug('unlock_db()') - self.qubes_store_file.close() - self.qubes_store_file = None - - def save(self): - self.log.debug('save()') - root = lxml.etree.Element( - "QubesVmCollection", - - default_template=str(self.default_template_qid) \ - if self.default_template_qid is not None else "None", - - default_netvm=str(self.default_netvm_qid) \ - if self.default_netvm_qid is not None else "None", - - default_fw_netvm=str(self.default_fw_netvm_qid) \ - if self.default_fw_netvm_qid is not None else "None", - - updatevm=str(self.updatevm_qid) \ - if self.updatevm_qid is not None else "None", - - clockvm=str(self.clockvm_qid) \ - if self.clockvm_qid is not None else "None", - - default_kernel=str(self.default_kernel) \ - if self.default_kernel is not None else "None", - ) - - for vm in self.values(): - element = vm.create_xml_element() - if element is not None: - root.append(element) - tree = lxml.etree.ElementTree(root) - - try: - - new_store_file = tempfile.NamedTemporaryFile(prefix=self.qubes_store_filename, delete=False) - if os.name == 'posix': - fcntl.lockf (new_store_file, fcntl.LOCK_EX) - elif os.name == 'nt': - overlapped = pywintypes.OVERLAPPED() - win32file.LockFileEx(win32file._get_osfhandle(new_store_file.fileno()), - win32con.LOCKFILE_EXCLUSIVE_LOCK, 0, -0x10000, overlapped) - tree.write(new_store_file, encoding="UTF-8", pretty_print=True) - new_store_file.flush() - os.chmod(new_store_file.name, 0660) - os.chown(new_store_file.name, -1, grp.getgrnam('qubes').gr_gid) - os.rename(new_store_file.name, self.qubes_store_filename) - self.qubes_store_file.close() - self.qubes_store_file = new_store_file - except EnvironmentError as err: - print("{0}: export error: {1}".format( - os.path.basename(sys.argv[0]), err)) - return False - return True - - def set_netvm_dependency(self, element): - kwargs = {} - attr_list = ("qid", "uses_default_netvm", "netvm_qid") - - for attribute in attr_list: - kwargs[attribute] = element.get(attribute) - - vm = self[int(kwargs["qid"])] - - if "uses_default_netvm" not in kwargs: - vm.uses_default_netvm = True - else: - vm.uses_default_netvm = ( - True if kwargs["uses_default_netvm"] == "True" else False) - if vm.uses_default_netvm is True: - if vm.is_proxyvm(): - netvm = self.get_default_fw_netvm() - else: - netvm = self.get_default_netvm() - kwargs.pop("netvm_qid") - else: - if kwargs["netvm_qid"] == "none" or kwargs["netvm_qid"] is None: - netvm = None - kwargs.pop("netvm_qid") - else: - netvm_qid = int(kwargs.pop("netvm_qid")) - if netvm_qid not in self: - netvm = None - else: - netvm = self[netvm_qid] - - # directly set internal attr to not call setters... - vm._netvm = netvm - if netvm: - netvm.connected_vms[vm.qid] = vm - - - def load_globals(self, element): - default_template = element.get("default_template") - self.default_template_qid = int(default_template) \ - if default_template.lower() != "none" else None - - default_netvm = element.get("default_netvm") - if default_netvm is not None: - self.default_netvm_qid = int(default_netvm) \ - if default_netvm != "None" else None - #assert self.default_netvm_qid is not None - - default_fw_netvm = element.get("default_fw_netvm") - if default_fw_netvm is not None: - self.default_fw_netvm_qid = int(default_fw_netvm) \ - if default_fw_netvm != "None" else None - #assert self.default_netvm_qid is not None - - updatevm = element.get("updatevm") - if updatevm is not None: - self.updatevm_qid = int(updatevm) \ - if updatevm != "None" else None - #assert self.default_netvm_qid is not None - - clockvm = element.get("clockvm") - if clockvm is not None: - self.clockvm_qid = int(clockvm) \ - if clockvm != "None" else None - - self.default_kernel = element.get("default_kernel") - - - def _check_global(self, attr, default): - qid = getattr(self, attr) - if qid is None: - return - try: - self[qid] - except KeyError: - setattr(self, attr, default) - - - def check_globals(self): - '''Ensure that all referenced qids are present in the collection''' - self._check_global('default_template_qid', None) - self._check_global('default_fw_netvm_qid', None) - self._check_global('default_netvm_qid', self.default_fw_netvm_qid) - self._check_global('updatevm_qid', self.default_netvm_qid) - self._check_global('clockvm_qid', self.default_netvm_qid) - - - def load(self): - self.log.debug('load()') - self.clear() - - try: - self.qubes_store_file.seek(0) - tree = lxml.etree.parse(self.qubes_store_file) - except (EnvironmentError, - xml.parsers.expat.ExpatError) as err: - print("{0}: import error: {1}".format( - os.path.basename(sys.argv[0]), err)) - return False - - self.load_globals(tree.getroot()) - - for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), - key=lambda _x: _x[1].load_order): - vms_of_class = tree.findall(vm_class_name) - # first non-template based, then template based - sorted_vms_of_class = sorted(vms_of_class, key= \ - lambda x: str(x.get('template_qid')).lower() != "none") - for element in sorted_vms_of_class: - try: - vm = vm_class(xml_element=element, collection=self) - self[vm.qid] = vm - except (ValueError, LookupError) as err: - print("{0}: import error ({1}): {2}".format( - os.path.basename(sys.argv[0]), vm_class_name, err)) - raise - return False - - # After importing all VMs, set netvm references, in the same order - for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), - key=lambda _x: _x[1].load_order): - for element in tree.findall(vm_class_name): - try: - self.set_netvm_dependency(element) - except (ValueError, LookupError) as err: - print("{0}: import error2 ({}): {}".format( - os.path.basename(sys.argv[0]), vm_class_name, err)) - return False - - self.check_globals() - - # if there was no clockvm entry in qubes.xml, try to determine default: - # root of default NetVM chain - if tree.getroot().get("clockvm") is None: - if self.default_netvm_qid is not None: - clockvm = self[self.default_netvm_qid] - # Find root of netvm chain - while clockvm.netvm is not None: - clockvm = clockvm.netvm - - self.clockvm_qid = clockvm.qid - - # Disable ntpd in ClockVM - to not conflict with ntpdate (both are - # using 123/udp port) - if self.clockvm_qid is not None: - self[self.clockvm_qid].services['ntpd'] = False - - # Add dom0 if wasn't present in qubes.xml - if not 0 in self.keys(): - dom0vm = QubesAdminVm (collection=self) - self[dom0vm.qid] = dom0vm - - return True - - def pop(self, qid): - self.log.debug('pop({})'.format(qid)) - - if self.default_netvm_qid == qid: - self.default_netvm_qid = None - if self.default_fw_netvm_qid == qid: - self.default_fw_netvm_qid = None - if self.clockvm_qid == qid: - self.clockvm_qid = None - if self.updatevm_qid == qid: - self.updatevm_qid = None - if self.default_template_qid == qid: - self.default_template_qid = None - - return super(QubesVmCollection, self).pop(qid) class QubesDaemonPidfile(object): def __init__(self, name): diff --git a/qubes/__init__.py b/qubes/__init__.py index 74793219..38fbf7ad 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -1,7 +1,12 @@ #!/usr/bin/python2 -O +# -*- coding: utf-8 -*- + +from __future__ import absolute_import ''' Qubes OS + +:copyright: © 2010-2014 Invisible Things Lab ''' __author__ = 'Invisible Things Lab' @@ -9,8 +14,42 @@ __license__ = 'GPLv2 or later' __version__ = 'R3' import ast +import atexit +import collections +import grp +import os +import os.path +import sys +import tempfile +import time +import warnings + +import __builtin__ + +import lxml.etree +import xml.parsers.expat + +if os.name == 'posix': + import fcntl +elif os.name == 'nt': + import win32con + import win32file + import pywintypes +else: + raise RuntimeError, "Qubes works only on POSIX or WinNT systems" + +import libvirt +try: + import xen.lowlevel.xs +except ImportError: + pass + +#: FIXME documentation +MAX_QID = 253 + +#: FIXME documentation +MAX_NETID = 253 -import qubes._pluginloader class QubesException(Exception): '''Exception that can be shown to the user''' @@ -24,7 +63,7 @@ class QubesVMMConnection(object): self._xc = None self._offline_mode = False - @property + @__builtin__.property def offline_mode(self): '''Check or enable offline mode (do not actually connect to vmm)''' return self._offline_mode @@ -58,13 +97,13 @@ class QubesVMMConnection(object): libvirt.registerErrorHandler(self._libvirt_error_handler, None) atexit.register(self._libvirt_conn.close) - @property + @__builtin__.property def libvirt_conn(self): '''Connection to libvirt''' self.init_vmm_connection() return self._libvirt_conn - @property + @__builtin__.property def xs(self): '''Connection to Xen Store @@ -90,12 +129,12 @@ class QubesHost(object): # print "QubesHost: free_mem = {0}".format (self.get_free_xen_memory()) # print "QubesHost: total_cpus = {0}".format (self.xen_no_cpus) - @property + @__builtin__.property def memory_total(self): '''Total memory, in bytes''' return self._total_mem - @property + @__builtin__.property def no_cpus(self): '''Noumber of CPUs''' return self._no_cpus @@ -141,7 +180,7 @@ class QubesHost(object): return (current_time, current) -class QubesVmLabel(object): +class Label(object): '''Label definition for virtual machines Label specifies colour of the padlock displayed next to VM's name. @@ -151,10 +190,9 @@ class QubesVmLabel(object): :param int index: numeric identificator of label :param str color: colour specification as in HTML (``#abcdef``) :param str name: label's name like "red" or "green" - :param bool dispvm: :py:obj:`True` if this is :py:class:`qubes.vm.dispvm.DispVM` label - ''' - def __init__(self, index, color, name, dispvm=False): + + def __init__(self, index, color, name): #: numeric identificator of label self.index = index @@ -164,65 +202,683 @@ class QubesVmLabel(object): #: label's name like "red" or "green" self.name = name - #: :py:obj:`True` if this is :py:class:`qubes.vm.dispvm.DispVM` label - self.dispvm = dispvm + #: freedesktop icon name, suitable for use in :py:meth:`PyQt4.QtGui.QIcon.fromTheme` + self.icon = 'appvm-' + name #: freedesktop icon name, suitable for use in :py:meth:`PyQt4.QtGui.QIcon.fromTheme` - self.icon = '{}-{}'.format(('dispvm' if dispvm else 'appvm'), name) + #: on DispVMs + self.icon_dispvm = 'dispvm-' + name + @classmethod def fromxml(cls, xml): '''Create label definition from XML node - :param :py:class:`lxml.etree._Element` xml: XML node reference - :rtype: :py:class:`qubes.QubesVmLabel` + :param lxml.etree._Element xml: XML node reference + :rtype: :py:class:`qubes.Label` ''' index = int(xml.get('id').split('-', 1)[1]) color = xml.get('color') name = xml.text - dispvm = ast.literal_eval(xml.get('dispvm', 'False')) - return cls(index, color, name, dispvm) + return cls(index, color, name) + + + def __xml__(self): + element = lxml.etree.Element('label', id='label-' + self.index, color=self.color) + element.text = self.name + return element + def __repr__(self): return '{}({!r}, {!r}, {!r}, dispvm={!r})'.format( self.__class__.__name__, self.index, self.color, - self.name, - self.dispvm) + self.name) - # self.icon_path is obsolete - # use QIcon.fromTheme(label.icon) where applicable - @property + + @__builtin__.property def icon_path(self): '''Icon path - DEPRECATED --- use :py:meth:`PyQt4.QtGui.QIcon.fromTheme` and :py:attr:`QubesVmLabel.icon`''' + .. deprecated:: 2.0 + use :py:meth:`PyQt4.QtGui.QIcon.fromTheme` and :py:attr:`icon` + ''' return os.path.join(system_path['qubes_icon_dir'], self.icon) + ".png" -#: Globally defined labels -QubesVmLabels = { - "red": QubesVmLabel(1, "0xcc0000", "red" ), - "orange": QubesVmLabel(2, "0xf57900", "orange" ), - "yellow": QubesVmLabel(3, "0xedd400", "yellow" ), - "green": QubesVmLabel(4, "0x73d216", "green" ), - "gray": QubesVmLabel(5, "0x555753", "gray" ), - "blue": QubesVmLabel(6, "0x3465a4", "blue" ), - "purple": QubesVmLabel(7, "0x75507b", "purple" ), - "black": QubesVmLabel(8, "0x000000", "black" ), -} -#: Globally defined labels for :py:class:`qubes.vm.dispvm.DispVM` s -QubesDispVmLabels = { - "red": QubesVmLabel(1, "0xcc0000", "red", dispvm=True), - "orange": QubesVmLabel(2, "0xf57900", "orange", dispvm=True), - "yellow": QubesVmLabel(3, "0xedd400", "yellow", dispvm=True), - "green": QubesVmLabel(4, "0x73d216", "green", dispvm=True), - "gray": QubesVmLabel(5, "0x555753", "gray", dispvm=True), - "blue": QubesVmLabel(6, "0x3465a4", "blue", dispvm=True), - "purple": QubesVmLabel(7, "0x75507b", "purple", dispvm=True), - "black": QubesVmLabel(8, "0x000000", "black", dispvm=True), -} + @__builtin__.property + def icon_path_dispvm(self): + '''Icon path + .. deprecated:: 2.0 + use :py:meth:`PyQt4.QtGui.QIcon.fromTheme` and :py:attr:`icon_dispvm` + ''' + return os.path.join(system_path['qubes_icon_dir'], self.icon_dispvm) + ".png" + + +class VMCollection(object): + '''A collection of Qubes VMs + + VMCollection supports ``in`` operator. You may test for ``qid``, ``name`` + and whole VM object's presence. + + Iterating over VMCollection will yield machine objects. + ''' + + def __init__(self, app): + self.app = app + self._dict = dict() + + + def __repr__(self): + return '<{} {!r}>'.format(self.__class__.__name__, list(sorted(self.keys()))) + + + def items(self): + '''Iterate over ``(qid, vm)`` pairs''' + for qid in self.qids(): + yield (qid, self[qid]) + + + def qids(self): + '''Iterate over all qids + + qids are sorted by numerical order. + ''' + + return iter(sorted(self._dict.keys())) + + keys = qids + + + def names(self): + '''Iterate over all names + + names are sorted by lexical order. + ''' + + return iter(sorted(vm.name for vm in self._dict.values())) + + + def vms(self): + '''Iterate over all machines + + vms are sorted by qid. + ''' + + return iter(sorted(self._dict.values())) + + __iter__ = vms + values = vms + + + def add(self, value): + '''Add VM to collection + + :param qubes.vm.BaseVM value: VM to add + :raises TypeError: when value is of wrong type + :raises ValueError: when there is already VM which has equal ``qid`` + ''' + + # XXX this violates duck typing, should we do it? + if not isinstance(value, qubes.vm.BaseVM): + raise TypeError('{} holds only BaseVM instances'.format(self.__class__.__name__)) + + if value.qid in self: + raise ValueError('This collection already holds VM that has qid={!r} (!r)'.format( + value.qid, self[value.qid])) + if value.name in self: + raise ValueError('This collection already holds VM that has name={!r} (!r)'.format( + value.name, self[value.name])) + + self._dict[value.qid] = value + + + def __getitem__(self, key): + if isinstance(key, int): + return self._dict[key] + + if isinstance(key, basestring): + for vm in self: + if (vm.name == key): + return vm + raise KeyError(key) + + if isinstance(key, qubes.vm.BaseVM): + if key in self: + return key + raise KeyError(key) + + raise KeyError(key) + + + def __delitem__(self, key): + del self._dict[self[key].qid] + + + def __contains__(self, key): + return any((key == vm or key == vm.qid or key == vm.name) for vm in self) + + + def __len__(self): + return len(self._dict) + + + def get_vms_based_on(self, template): + template = self[template] + return set(vm for vm in self if vm.template == template) + + + def get_vms_connected_to(self, netvm): + new_vms = set([netvm]) + dependend_vms = set() + + # Dependency resolving only makes sense on NetVM (or derivative) +# if not self[netvm_qid].is_netvm(): +# return set([]) + + while len(new_vms) > 0: + cur_vm = new_vms.pop() + for vm in cur_vm.connected_vms.values(): + if vm in dependend_vms: + continue + dependend_vms.add(vm.qid) +# if vm.is_netvm(): + new_vms.append(vm.qid) + + return dependent_vms + + + # XXX with Qubes Admin Api this will probably lead to race condition + # whole process of creating and adding should be synchronised + def get_new_unused_qid(self): + used_ids = set(self.qids()) + for i in range(1, MAX_QID): + if i not in used_ids: + return i + raise LookupError("Cannot find unused qid!") + + + def get_new_unused_netid(self): + used_ids = set([vm.netid for vm in self]) # if vm.is_netvm()]) + for i in range(1, MAX_NETID): + if i not in used_ids: + return i + raise LookupError("Cannot find unused netid!") + + +class property(object): + '''Qubes property. + + This class holds one property that can be saved to and loaded from + :file:`qubes.xml`. It is used for both global and per-VM properties. + + :param str name: name of the property + :param collections.Callable setter: if not :py:obj:`None`, this is used to initialise value; first parameter to the function is holder instance and the second is value; this is called before ``type`` + :param type type: if not :py:obj:`None`, value is coerced to this type + :param object default: default value + :param int load_stage: stage when property should be loaded (see :py:class:`Qubes` for description of stages) + :param int order: order of evaluation (bigger order values are later) + :param str doc: docstring; you may use RST markup + + ''' + + def __init__(self, name, setter=None, type=None, default=None, + load_stage=2, order=0, save_via_ref=False, doc=None): + self.__name__ = name + self._setter = setter + self._type = type + self._default = default + self.order = order + self.load_stage = load_stage + self.save_via_ref = save_via_ref + self.__doc__ = doc + self._attr_name = '_qubesprop_' + name + + + def __get__(self, instance, owner): +# sys.stderr.write('{!r}.__get__({}, {!r})\n'.format(self.__name__, hex(id(instance)), owner)) + if instance is None: + return self + + # XXX this violates duck typing, shall we keep it? + if not isinstance(instance, PropertyHolder): + raise AttributeError( + 'qubes.property should be used on qubes.PropertyHolder instances only') + +# sys.stderr.write(' __get__ try\n') + try: + return getattr(instance, self._attr_name) + + except AttributeError: +# sys.stderr.write(' __get__ except\n') + if self._default is None: + raise AttributeError('property {!r} not set'.format(self.__name__)) + elif isinstance(self._default, collections.Callable): + return self._default(instance) + else: + return self._default + + + def __set__(self, instance, value): + if self._setter is not None: + value = self._setter(instance, self, value) + if self._type is not None: + value = self._type(value) + instance._init_property(self, value) + + + def __repr__(self): + return '<{} object at {:#x} name={!r} default={!r}>'.format( + self.__class__.__name__, id(self), self.__name__, self._default) + + + def __hash__(self): + return hash(self.__name__) + + + def __eq__(self, other): + return self.__name__ == other.__name__ + + + # + # some setters provided + # + + @staticmethod + def forbidden(self, prop, value): + '''Property setter that forbids loading a property + + This is used to effectively disable property in classes which inherit + unwanted property. When someone attempts to load such a property, it + + :throws AttributeError: always + ''' + + raise AttributeError('setting {} property on {} instance is forbidden'.format( + prop.__name__, self.__class__.__name__)) + + +class PropertyHolder(object): + '''Abstract class for holding :py:class:`qubes.property`''' + + def __init__(self, xml, *args, **kwargs): + super(PropertyHolder, self).__init__(*args, **kwargs) + self.xml = xml + + + def get_props_list(self, load_stage=None): + '''List all properties attached to this VM + + :param load_stage: Filter by load stage + :type load_stage: :py:func:`int` or :py:obj:`None` + ''' + +# sys.stderr.write('{!r}.get_props_list(load_stage={})\n'.format('self', load_stage)) + props = set() + for class_ in self.__class__.__mro__: + props.update(prop for prop in class_.__dict__.values() + if isinstance(prop, property)) + if load_stage is not None: + props = set(prop for prop in props + if prop.load_stage == load_stage) +# sys.stderr.write(' props={!r}\n'.format(props)) + return sorted(props, key=lambda prop: (prop.load_stage, prop.order, prop.__name__)) + + + def _init_property(self, prop, value): + '''Initialise property to a given value, without side effects. + + :param qubes.property prop: property object of particular interest + :param value: value + ''' + + setattr(self, prop._attr_name, value) + + + def load_properties(self, load_stage=None): + '''Load properties from immediate children of XML node. + + :param lxml.etree._Element xml: XML node reference + ''' + +# sys.stderr.write('<{}>.load_properties(load_stage={}) xml={!r}\n'.format(hex(id(self)), load_stage, self.xml)) + + all_names = set(prop.__name__ for prop in self.get_props_list(load_stage)) +# sys.stderr.write(' all_names={!r}\n'.format(all_names)) + for node in self.xml.xpath('./properties/property'): + name = node.get('name') + value = node.get('ref') or node.text + +# sys.stderr.write(' load_properties name={!r} value={!r}\n'.format(name, value)) + if not name in all_names: + raise AttributeError( + 'No property {!r} found in {!r}'.format( + name, self.__class__)) + + setattr(self, name, value) +# sys.stderr.write(' load_properties return\n') + + + def save_properties(self, with_defaults=False): + '''Iterator that yields XML nodes representing set properties. + + :param bool with_defaults: If :py:obj:`True`, then it also includes properties which were not set explicite, but have default values filled. + ''' + +# sys.stderr.write('{!r}.save_properties(with_defaults={})\n'.format(self, with_defaults)) + + properties = lxml.etree.Element('properties') + + for prop in self.get_props_list(): + try: + value = str(getattr(self, (prop.__name__ if with_defaults else prop._attr_name))) + except AttributeError, e: +# sys.stderr.write('AttributeError: {!s}\n'.format(e)) + continue + + element = lxml.etree.Element('property', name=prop.__name__) + if prop.save_via_ref: + element.set('ref', value) + else: + element.text = value + properties.append(element) + + return properties + + +import qubes.vm.qubesvm +import qubes.vm.templatevm + + +class VMProperty(property): + '''Property that is referring to a VM + + :param type vmclass: class that returned VM is supposed to be instance of + + and all supported by :py:class:`property` with the exception of ``type`` and ``setter`` + ''' + + def __init__(self, name, vmclass=qubes.vm.BaseVM, **kwargs): + if 'type' in kwargs: + raise TypeError("'type' keyword parameter is unsupported in {}".format( + self.__class__.__name__)) + if 'setter' in kwargs: + raise TypeError("'setter' keyword parameter is unsupported in {}".format( + self.__class__.__name__)) + super(VMProperty, self).__init__(name, **kwargs) + self.vmclass = vmclass + + + def __set__(self, instance, value): + vm = instance.app.domains[value] + if not isinstance(vm, self.vmclass): + raise TypeError('wrong VM class: domains[{!r}] if of type {!s} and not {!s}'.format( + value, vm.__class__.__name__, self.vmclass.__name__)) + + super(VMProperty, self).__set__(self, instance, vm) + + +class Qubes(PropertyHolder): + '''Main Qubes application + + :param str store: path to ``qubes.xml`` + + The store is loaded in stages. + + In the first stage there are loaded some basic features from store + (currently labels). + + In the second stage stubs for all VMs are loaded. They are filled with + their basic properties, like ``qid`` and ``name``. + + In the third stage all global properties are loaded. They often reference + VMs, like default netvm, so they should be filled after loading VMs. + + In the fourth stage all remaining VM properties are loaded. They also need + all VMs loaded, because they represent dependencies between VMs like + aforementioned netvm. + + In the fifth stage there are some fixups to ensure sane system operation. + ''' + + default_netvm = VMProperty('default_netvm', load_stage=3, + doc='Default NetVM for new AppVMs') + default_fw_netvm = VMProperty('default_fw_netvm', load_stage=3, + doc='Default NetVM for new ProxyVMs') + default_template = VMProperty('default_template', load_stage=3, + vmclass=qubes.vm.templatevm.TemplateVM, + doc='Default template for new AppVMs') + updatevm = VMProperty('updatevm', load_stage=3, + doc='Which VM to use as ``yum`` proxy for updating AdminVM and TemplateVMs') + clockvm = VMProperty('clockvm', load_stage=3, + doc='Which VM to use as NTP proxy for updating AdminVM') + default_kernel = property('default_kernel', load_stage=3, + doc='Which kernel to use when not overriden in VM') + + + def __init__(self, store='/var/lib/qubes/qubes.xml'): + #: collection of all VMs managed by this Qubes instance + self.domains = VMCollection() + + #: collection of all available labels for VMs + self.labels = {} + + self._store = store + + try: + self.load() + except IOError: + self._init() + + super(PropertyHolder, self).__init__(xml=lxml.etree.parse(self.qubes_store_file)) + + + def _open_store(self): + if hasattr(self, '_storefd'): + return + + self._storefd = open(self._store, 'r+') + + if os.name == 'posix': + fcntl.lockf (self.qubes_store_file, fcntl.LOCK_EX) + elif os.name == 'nt': + overlapped = pywintypes.OVERLAPPED() + win32file.LockFileEx(win32file._get_osfhandle(self.qubes_store_file.fileno()), + win32con.LOCKFILE_EXCLUSIVE_LOCK, 0, -0x10000, overlapped) + + + def load(self): + ''' + :throws EnvironmentError: failure on parsing store + :throws xml.parsers.expat.ExpatError: failure on parsing store + ''' + self._open_store() + + # stage 1: load labels + for node in self._xml.xpath('./labels/label'): + label = Label.fromxml(node) + self.labels[label.id] = label + + # stage 2: load VMs + for node in self._xml.xpath('./domains/domain'): + cls = qubes.vm.load(node.get("class")) + vm = cls.fromxml(self, node) + self.domains.add(vm) + + if not 0 in self.domains: + self.domains.add(qubes.vm.adminvm.AdminVM(self)) + + # stage 3: load global properties + self.load_properties(self.xml, load_stage=3) + + # stage 4: fill all remaining VM properties + for vm in self.domains: + vm.load_properties(None, load_stage=4) + + # stage 5: misc fixups + + # if we have no default netvm, make first one the default + if not hasattr(self, 'default_netvm'): + for vm in self.domains: + if hasattr(vm, 'provides_network') and hasattr(vm, 'netvm'): + self.default_netvm = vm + break + + if not hasattr(self, 'default_fw_netvm'): + for vm in self.domains: + if hasattr(vm, 'provides_network') and not hasattr(vm, 'netvm'): + self.default_netvm = vm + break + + # first found template vm is the default + if not hasattr(self, 'default_template'): + for vm in self.domains: + if isinstance(vm, qubes.vm.templatevm.TemplateVM): + self.default_template = vm + break + + # if there was no clockvm entry in qubes.xml, try to determine default: + # root of default NetVM chain + if not hasattr(self, 'clockvm') and hasattr(self, 'default_netvm'): + clockvm = self.default_netvm + # Find root of netvm chain + while clockvm.netvm is not None: + clockvm = clockvm.netvm + + self.clockvm = clockvm + + # Disable ntpd in ClockVM - to not conflict with ntpdate (both are + # using 123/udp port) + if hasattr(self, 'clockvm'): + self.clockvm.services['ntpd'] = False + + + def _init(self): + self._open_store() + + self.labels = { + 1: Label(1, '0xcc0000', 'red'), + 2: Label(2, '0xf57900', 'orange'), + 3: Label(3, '0xedd400', 'yellow'), + 4: Label(4, '0x73d216', 'green'), + 5: Label(5, '0x555753', 'gray'), + 6: Label(6, '0x3465a4', 'blue'), + 7: Label(7, '0x75507b', 'purple'), + 8: Label(8, '0x000000', 'black'), + } + + + def __del__(self): + # intentionally do not call explicit unlock to not unlock the file + # before all buffers are flushed + self._storefd.close() + del self._storefd + + + def __xml__(self): + element = lxml.etree.Element('qubes') + + element.append(self.save_labels()) + element.append(self.save_properties()) + + domains = lxml.etree.Element('domains') + for vm in self.domains: + domains.append(vm.__xml__()) + element.append(domains) + + return element + + + def save(self): + '''Save all data to qubes.xml + ''' + self._storefd.seek(0) + self._storefd.truncate() + lxml.etree.ElementTree(self.__xml__()).write( + self._storefd, encoding='utf-8', pretty_print=True) + self._storefd.sync() + os.chmod(self._store, 0660) + os.chown(self._store, -1, grp.getgrnam('qubes').gr_gid) + + + def save_labels(self): + '''Serialise labels + + :rtype: lxml.etree._Element + ''' + + labels = lxml.etree.Element('labels') + for label in self.labels: + labels.append(label.__xml__()) + return labels + + + def add_new_vm(self, vm): + '''Add new Virtual Machine to colletion + + ''' + + if not hasattr(vm, 'qid'): + vm.qid = self.domains.get_new_unused_qid() + + self.domains.add(vm) + + # + # XXX + # all this will be moved to an event handler + # and deduplicated with self.load() + # + + # make first created NetVM the default one + if not hasattr(self, 'default_fw_netvm') \ + and vm.provides_network \ + and not hasattr(vm, 'netvm'): + self.default_fw_netvm = vm + + if not hasattr(self, 'default_netvm') \ + and vm.provides_network \ + and hasattr(vm, 'netvm'): + self.default_netvm = vm + + # make first created TemplateVM the default one + if not hasattr(self, 'default_template') \ + and not hasattr(vm, 'template'): + self.default_template = vm + + # make first created ProxyVM the UpdateVM + if not hasattr(self, 'default_netvm') \ + and vm.provides_network \ + and hasattr(vm, 'netvm'): + self.updatevm = vm + + # by default ClockVM is the first NetVM + if not hasattr(self, 'clockvm') \ + and vm.provides_network \ + and hasattr(vm, 'netvm'): + self.default_clockvm = vm + + # XXX don't know if it should return self + return vm + + # XXX This was in QubesVmCollection, will be in an event +# def pop(self, qid): +# if self.default_netvm_qid == qid: +# self.default_netvm_qid = None +# if self.default_fw_netvm_qid == qid: +# self.default_fw_netvm_qid = None +# if self.clockvm_qid == qid: +# self.clockvm_qid = None +# if self.updatevm_qid == qid: +# self.updatevm_qid = None +# if self.default_template_qid == qid: +# self.default_template_qid = None +# +# return super(QubesVmCollection, self).pop(qid) + + +# load plugins +import qubes._pluginloader diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index dc7423a5..7c7ff150 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -8,9 +8,6 @@ Main public classes .. autoclass:: BaseVM :members: :show-inheritance: -.. autoclass:: property - :members: - :show-inheritance: Helper classes and functions ---------------------------- @@ -57,57 +54,11 @@ import functools import sys import dateutil.parser +import lxml.etree +import qubes import qubes.plugins -class property(object): - '''Qubes VM property. - - This class holds one property that can be saved and loaded from qubes.xml - - :param str name: name of the property - :param object default: default value - :param type type: if not :py:obj:`None`, this is used to initialise value - :param int order: order of evaluation (bigger order values are later) - :param str doc: docstring - - ''' - - def __init__(self, name, default=None, type=None, order=0, doc=None): - self.__name__ = name - self._default = default - self._type = type - self.order = order - self.__doc__ = doc - - self._attr_name = '_qubesprop_' + self.__name__ - - def __get__(self, instance, owner): - if instance is None: - return self - - try: - return getattr(instance, self._attr_name) - - except AttributeError: - if self._default is None: - raise AttributeError('property not set') - else: - return self._default - - def __set__(self, instance, value): - setattr(instance, self._attr_name, - (self._type(value) if self._type is not None else value)) - - def __repr__(self): - return '<{} object at {:#x} name={!r} default={!r}>'.format( - self.__class__.__name__, id(self), self.__name__, self._default) - - def __hash__(self): - return hash(self.__name__) - - def __eq__(self, other): - return self.__name__ == other.__name__ class VMPlugin(qubes.plugins.Plugin): '''Metaclass for :py:class:`.BaseVM`''' @@ -115,7 +66,8 @@ class VMPlugin(qubes.plugins.Plugin): super(VMPlugin, cls).__init__(name, bases, dict_) cls.__hooks__ = collections.defaultdict(list) -class BaseVM(object): + +class BaseVM(qubes.PropertyHolder): '''Base class for all VMs :param xml: xml node from which to deserialise @@ -128,53 +80,132 @@ class BaseVM(object): __metaclass__ = VMPlugin - def get_props_list(self): - '''List all properties attached to this VM''' - props = set() - for class_ in self.__class__.__mro__: - props.update(prop for prop in class_.__dict__.values() - if isinstance(prop, property)) - return sorted(props, key=lambda prop: (prop.order, prop.__name__)) + def __init__(self, app, xml=None, load_stage=2, services={}, devices=None, tags={}, **kwargs): + self.app = app + self.xml = xml + self.services = services + self.devices = collections.defaultdict(list) if devices is None else devices + self.tags = tags - def __init__(self, xml): - self._xml = xml - - self.services = {} - self.devices = collections.defaultdict(list) - self.tags = {} - - if self._xml is None: - return - - # properties - all_names = set(prop.__name__ for prop in self.get_props_list()) - for node in self._xml.xpath('.//property'): - name = node.get('name') - value = node.get('ref') or node.text - - if not name in all_names: + all_names = set(prop.__name__ for prop in self.get_props_list(load_stage=2)) + for key in kwargs: + if not key in all_names: raise AttributeError( 'No property {!r} found in {!r}'.format( - name, self.__class__)) + key, self.__class__)) + setattr(self, key, kwargs[key]) - setattr(self, name, value) - # tags - for node in self._xml.xpath('.//tag'): - self.tags[node.get('name')] = node.text + def add_new_vm(self, vm): + '''Add new Virtual Machine to colletion + + ''' + + vm_cls = QubesVmClasses[vm_type] + if 'template' in kwargs: + if not vm_cls.is_template_compatible(kwargs['template']): + raise QubesException("Template not compatible with selected " + "VM type") + + vm = vm_cls(qid=qid, collection=self, **kwargs) + if not self.verify_new_vm(vm): + raise QubesException("Wrong VM description!") + self[vm.qid] = vm + + # make first created NetVM the default one + if self.default_fw_netvm_qid is None and vm.is_netvm(): + self.set_default_fw_netvm(vm) + + if self.default_netvm_qid is None and vm.is_proxyvm(): + self.set_default_netvm(vm) + + # make first created TemplateVM the default one + if self.default_template_qid is None and vm.is_template(): + self.set_default_template(vm) + + # make first created ProxyVM the UpdateVM + if self.updatevm_qid is None and vm.is_proxyvm(): + self.set_updatevm_vm(vm) + + # by default ClockVM is the first NetVM + if self.clockvm_qid is None and vm.is_netvm(): + self.set_clockvm_vm(vm) + + return vm + + @classmethod + def fromxml(cls, app, xml, load_stage=2): + '''Create VM from XML node + + :param qubes.Qubes app: :py:class:`qubes.Qubes` application instance + :param lxml.etree._Element xml: XML node reference + :param int load_stage: do not change the default (2) unless you know, what you are doing + ''' + +# sys.stderr.write('{}.fromxml(app={!r}, xml={!r}, load_stage={})\n'.format( +# cls.__name__, app, xml, load_stage)) + if xml is None: + return cls(app) + + services = {} + devices = collections.defaultdict(list) + tags = {} # services - for node in self._xml.xpath('.//service'): - self.services[node.text] = bool(ast.literal_eval(node.get('enabled', 'True'))) + for node in xml.xpath('./services/service'): + services[node.text] = bool(ast.literal_eval(node.get('enabled', 'True'))) # devices (pci, usb, ...) - for parent in self._xml.xpath('.//devices'): + for parent in xml.xpath('./devices'): devclass = parent.get('class') for node in parent.xpath('./device'): - self.devices[devclass].append(node.text) + devices[devclass].append(node.text) - # firewall - #TODO + # tags + for node in xml.xpath('./tags/tag'): + tags[node.get('name')] = node.text + + # properties + self = cls(app, xml=xml, services=services, devices=devices, tags=tags) + self.load_properties(load_stage=load_stage) + + # TODO: firewall, policy + +# sys.stderr.write('{}.fromxml return\n'.format(cls.__name__)) + return self + + + def __xml__(self): + element = lxml.etree.Element('domain', id='domain-' + str(self.qid)) + + element.append(self.save_properties()) + + services = lxml.etree.Element('services') + for service in self.services: + node = lxml.etree.Element('service') + node.text = service + if not self.services[service]: + node.set('enabled', 'False') + services.append(node) + element.append(services) + + for devclass in self.devices: + devices = lxml.etree.Element('devices') + devices.set('class', devclass) + for device in self.devices[devclass]: + node = lxml.etree.Element('device') + node.text = device + devices.append(node) + element.append(devices) + + tags = lxml.etree.Element('tags') + for tag in self.tags: + node = lxml.etree.Element('tag', name=tag) + node.text = self.tags[tag] + tags.append(node) + element.append(tags) + + return element def __repr__(self): return '<{} object at {:#x} {}>'.format( @@ -182,6 +213,7 @@ class BaseVM(object): ' '.join('{}={}'.format(prop.__name__, getattr(self, prop.__name__)) for prop in self.get_props_list())) + @classmethod def add_hook(cls, event, f): '''Add hook to entire VM class and all subclasses @@ -195,6 +227,7 @@ class BaseVM(object): cls.__hooks__[event].append(f) + def fire_hooks(self, event, *args, **kwargs): '''Fire hooks associated with an event diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 95b030dc..99b5d755 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -1,8 +1,22 @@ #!/usr/bin/python2 -O +import qubes import qubes.vm class QubesVM(qubes.vm.BaseVM): '''Base functionality of Qubes VM shared between all VMs.''' - def __init__(self, D): - super(QubesVM, self).__init__(D) + + label = qubes.property('label', + setter=(lambda self, prop, value: self.app.labels[int(value.rsplit('-', 1)[1])]), + doc='Colourful label assigned to VM. This is where you set the colour of the padlock.') + + netvm = qubes.property('netvm', load_stage=4, + default=(lambda self: self.app.default_fw_netvm if self.provides_network + else self.app.default_fw_netvm), + doc='VM that provides network connection to this domain. ' + 'When :py:obj:`False`, machine is disconnected. ' + 'When :py:obj:`None` (or absent), domain uses default NetVM.') + + provides_network = qubes.property('provides_network', + type=bool, + doc=':py:obj:`True` if it is NetVM or ProxyVM, false otherwise') diff --git a/qubes/vm/templatevm.py b/qubes/vm/templatevm.py index d1df2495..ec92bdff 100644 --- a/qubes/vm/templatevm.py +++ b/qubes/vm/templatevm.py @@ -1,8 +1,13 @@ #!/usr/bin/python2 -O +import qubes import qubes.vm.qubesvm class TemplateVM(qubes.vm.qubesvm.QubesVM): '''Template for AppVM''' + + template = qubes.property('template', + setter=qubes.property.forbidden) + def __init__(self, D): super(TemplateVM, self).__init__(D) diff --git a/tests/init.py b/tests/init.py index c91e65c4..13d4c9c9 100644 --- a/tests/init.py +++ b/tests/init.py @@ -7,9 +7,19 @@ import lxml.etree sys.path.insert(0, '../') import qubes +import qubes.vm -class TC_QubesVmLabel(unittest.TestCase): - def test_000_appvm(self): +class TC_10_Label(unittest.TestCase): + def test_000_constructor(self): + label = qubes.Label(1, '#cc0000', 'red') + + self.assertEqual(label.index, 1) + self.assertEqual(label.color, '#cc0000') + self.assertEqual(label.name, 'red') + self.assertEqual(label.icon, 'appvm-red') + self.assertEqual(label.icon_dispvm, 'dispvm-red') + + def test_001_fromxml(self): xml = lxml.etree.XML(''' @@ -19,25 +29,175 @@ class TC_QubesVmLabel(unittest.TestCase): ''') node = xml.xpath('//label')[0] - label = qubes.QubesVmLabel.fromxml(node) + label = qubes.Label.fromxml(node) self.assertEqual(label.index, 1) self.assertEqual(label.color, '#cc0000') self.assertEqual(label.name, 'red') - self.assertEqual(label.dispvm, False) self.assertEqual(label.icon, 'appvm-red') + self.assertEqual(label.icon_dispvm, 'dispvm-red') - def test_001_dispvm(self): + +class TestHolder(qubes.PropertyHolder): + testprop1 = qubes.property('testprop1', order=0) + testprop2 = qubes.property('testprop2', order=1, save_via_ref=True) + testprop3 = qubes.property('testprop3', order=2, default='testdefault') + testprop4 = qubes.property('testprop4', order=3) + +class TC_00_PropertyHolder(unittest.TestCase): + def assertXMLEqual(self, xml1, xml2): + self.assertEqual(xml1.tag, xml2.tag) + self.assertEqual(xml1.text, xml2.text) + self.assertEqual(sorted(xml1.keys()), sorted(xml2.keys())) + for key in xml1.keys(): + self.assertEqual(xml1.get(key), xml2.get(key)) + + def setUp(self): xml = lxml.etree.XML(''' - - - + + testvalue1 + + ''') - node = xml.xpath('//label')[0] - label = qubes.QubesVmLabel.fromxml(node) + self.holder = TestHolder(xml) - self.assertEqual(label.dispvm, True) - self.assertEqual(label.icon, 'dispvm-red') + def test_000_load_properties(self): + self.holder.load_properties() + self.assertEquals(self.holder.testprop1, 'testvalue1') + self.assertEquals(self.holder.testprop2, 'testref2') + self.assertEquals(self.holder.testprop3, 'testdefault') + + with self.assertRaises(AttributeError): + self.holder.testprop4 + + def test_001_save_properties(self): + self.holder.load_properties() + + elements = self.holder.save_properties() + elements_with_defaults = self.holder.save_properties(with_defaults=True) + + self.assertEqual(len(elements), 2) + self.assertEqual(len(elements_with_defaults), 3) + + expected_prop1 = lxml.etree.Element('property', name='testprop1') + expected_prop1.text = 'testvalue1' + self.assertXMLEqual(elements_with_defaults[0], expected_prop1) + + expected_prop2 = lxml.etree.Element('property', name='testprop2', ref='testref2') + self.assertXMLEqual(elements_with_defaults[1], expected_prop2) + + expected_prop3 = lxml.etree.Element('property', name='testprop3') + expected_prop3.text = 'testdefault' + self.assertXMLEqual(elements_with_defaults[2], expected_prop3) + + +class TestVM(qubes.vm.BaseVM): + qid = qubes.property('qid', type=int) + name = qubes.property('name') + netid = qid + +class TC_11_VMCollection(unittest.TestCase): + def setUp(self): + # XXX passing None may be wrong in the future + self.vms = qubes.VMCollection(None) + + self.testvm1 = TestVM(None, qid=1, name='testvm1') + self.testvm2 = TestVM(None, qid=2, name='testvm2') + + def test_000_contains(self): + self.vms._dict = {1: self.testvm1} + + self.assertIn(1, self.vms) + self.assertIn('testvm1', self.vms) + self.assertIn(self.testvm1, self.vms) + + self.assertNotIn(2, self.vms) + self.assertNotIn('testvm2', self.vms) + self.assertNotIn(self.testvm2, self.vms) + + def test_001_getitem(self): + self.vms._dict = {1: self.testvm1} + + self.assertIs(self.vms[1], self.testvm1) + self.assertIs(self.vms['testvm1'], self.testvm1) + self.assertIs(self.vms[self.testvm1], self.testvm1) + + def test_002_add(self): + self.vms.add(self.testvm1) + self.assertIn(1, self.vms) + + with self.assertRaises(TypeError): + self.vms.add(object()) + + testvm_qid_collision = TestVM(None, name='testvm2', qid=1) + testvm_name_collision = TestVM(None, name='testvm1', qid=2) + + with self.assertRaises(ValueError): + self.vms.add(testvm_qid_collision) + with self.assertRaises(ValueError): + self.vms.add(testvm_name_collision) + + def test_003_qids(self): + self.vms.add(self.testvm1) + self.vms.add(self.testvm2) + + self.assertItemsEqual(self.vms.qids(), [1, 2]) + self.assertItemsEqual(self.vms.keys(), [1, 2]) + + def test_004_names(self): + self.vms.add(self.testvm1) + self.vms.add(self.testvm2) + + self.assertItemsEqual(self.vms.names(), ['testvm1', 'testvm2']) + + def test_005_vms(self): + self.vms.add(self.testvm1) + self.vms.add(self.testvm2) + + self.assertItemsEqual(self.vms.vms(), [self.testvm1, self.testvm2]) + self.assertItemsEqual(self.vms.values(), [self.testvm1, self.testvm2]) + + def test_006_items(self): + self.vms.add(self.testvm1) + self.vms.add(self.testvm2) + + self.assertItemsEqual(self.vms.items(), [(1, self.testvm1), (2, self.testvm2)]) + + def test_007_len(self): + self.vms.add(self.testvm1) + self.vms.add(self.testvm2) + + self.assertEqual(len(self.vms), 2) + + def test_008_delitem(self): + self.vms.add(self.testvm1) + self.vms.add(self.testvm2) + + del self.vms['testvm2'] + + self.assertItemsEqual(self.vms.vms(), [self.testvm1]) + + def test_100_get_new_unused_qid(self): + self.vms.add(self.testvm1) + self.vms.add(self.testvm2) + + self.vms.get_new_unused_qid() + + def test_101_get_new_unused_netid(self): + self.vms.add(self.testvm1) + self.vms.add(self.testvm2) + + self.vms.get_new_unused_netid() + +# def test_200_get_vms_based_on(self): +# pass + +# def test_201_get_vms_connected_to(self): +# pass + + +class TC_20_Qubes(unittest.TestCase): + pass diff --git a/tests/vm.py b/tests/vm.py index 273875e8..3b66a491 100644 --- a/tests/vm.py +++ b/tests/vm.py @@ -8,10 +8,13 @@ import lxml.etree sys.path.insert(0, '../') import qubes.vm + class TestVM(qubes.vm.BaseVM): - testprop = qubes.vm.property('testprop') - testlabel = qubes.vm.property('testlabel') - defaultprop = qubes.vm.property('defaultprop', default='defaultvalue') + qid = qubes.property('qid', type=int) + name = qubes.property('name') + testprop = qubes.property('testprop') + testlabel = qubes.property('testlabel') + defaultprop = qubes.property('defaultprop', default='defaultvalue') class TC_BaseVM(unittest.TestCase): def setUp(self): @@ -24,6 +27,8 @@ class TC_BaseVM(unittest.TestCase): + 1 + domain1 testvalue @@ -54,8 +59,10 @@ class TC_BaseVM(unittest.TestCase): def test_000_BaseVM_load(self): node = self.xml.xpath('//domain')[0] - vm = TestVM(node) + vm = TestVM.fromxml(None, node) + self.assertEqual(vm.qid, 1) + self.assertEqual(vm.testprop, 'testvalue') self.assertEqual(vm.testprop, 'testvalue') self.assertEqual(vm.testlabel, 'label-1') self.assertEqual(vm.defaultprop, 'defaultvalue') @@ -67,6 +74,8 @@ class TC_BaseVM(unittest.TestCase): 'disabledservice': False, }) + lxml.etree.ElementTree(vm.__xml__()).write(sys.stderr, encoding='utf-8', pretty_print=True) + def test_001_BaseVM_nxproperty(self): xml = lxml.etree.XML(''' @@ -82,7 +91,5 @@ class TC_BaseVM(unittest.TestCase): node = xml.xpath('//domain')[0] - def f(): - vm = TestVM(node) - - self.assertRaises(AttributeError, f) + with self.assertRaises(AttributeError): + TestVM.fromxml(None, node) From 855a434879198eeb98569a9d7bdcb1258120f7b9 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Tue, 9 Dec 2014 14:14:24 +0100 Subject: [PATCH 0020/1004] core3: event framework adjusted for global Qubes object From now, global events are emitted by qubes.Qubes object and handlers are registered there. --- qubes/__init__.py | 5 +++ qubes/_pluginloader.py | 3 -- qubes/events.py | 96 ++++++++++++++++++++++++++++-------------- qubes/ext/__init__.py | 52 +++++++++++++++++++---- qubes/vm/__init__.py | 21 +++++---- tests/events.py | 37 ++++++++++++++++ tests/init.py | 8 ++-- 7 files changed, 167 insertions(+), 55 deletions(-) create mode 100644 tests/events.py diff --git a/qubes/__init__.py b/qubes/__init__.py index 38fbf7ad..524223b0 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -29,6 +29,9 @@ import __builtin__ import lxml.etree import xml.parsers.expat +import qubes.ext + + if os.name == 'posix': import fcntl elif os.name == 'nt': @@ -661,6 +664,8 @@ class Qubes(PropertyHolder): def __init__(self, store='/var/lib/qubes/qubes.xml'): + self._extensions = set(ext(self) for ext in qubes.ext.Extension.register.values()) + #: collection of all VMs managed by this Qubes instance self.domains = VMCollection() diff --git a/qubes/_pluginloader.py b/qubes/_pluginloader.py index 4c9c7a58..fe333d56 100644 --- a/qubes/_pluginloader.py +++ b/qubes/_pluginloader.py @@ -1,6 +1,3 @@ from qubes.vm import * from qubes.ext import * -import qubes.ext - -qubes.ext.init() diff --git a/qubes/events.py b/qubes/events.py index e1fe13fb..c8cad69b 100644 --- a/qubes/events.py +++ b/qubes/events.py @@ -9,37 +9,28 @@ etc. import collections -import qubes.vm -#: collection of system-wide hooks -system_hooks = collections.defaultdict(list) - -def hook(event, vm=None, system=False): - '''Decorator factory. +def handler(event): + '''Event handler decorator factory. To hook an event, decorate a method in your plugin class with this decorator. + .. note:: + For hooking events from extensions, see :py:func:`qubes.ext.handler`. + :param str event: event type - :param type vm: VM to hook (leave as None to hook all VMs) - :param bool system: when :py:obj:`True`, hook is system-wide (not attached to any VM) ''' def decorator(f): - f.ho_event = event - - if system: - f.ho_vm = None - elif vm is None: - f.ho_vm = qubes.vm.BaseVM - else: - f.ho_vm = vm - + f.ha_event = event + f.ha_bound = True return f return decorator -def ishook(o): + +def ishandler(o): '''Test if a method is hooked to an event. :param object o: suspected hook @@ -48,24 +39,67 @@ def ishook(o): ''' return callable(o) \ - and hasattr(o, 'ho_event') \ - and hasattr(o, 'ho_vm') + and hasattr(o, 'ha_event') -def add_system_hook(event, f): - '''Add system-wide hook. - :param callable f: function to call +class EmitterMeta(type): + '''Metaclass for :py:class:`Emitter`''' + def __init__(cls, name, bases, dict_): + super(type, cls).__init__(name, bases, dict_) + cls.__handlers__ = collections.defaultdict(set) + + +class Emitter(object): + '''Subject that can emit events ''' - global_hooks[event].append(f) + __metaclass__ = EmitterMeta -def fire_system_hooks(event, *args, **kwargs): - '''Fire system-wide hooks. + def __init__(self, *args, **kwargs): + super(Emitter, self).__init__(*args, **kwargs) + try: + propnames = set(prop.__name__ for prop in self.get_props_list()) + except AttributeError: + propnames = set() - :param str event: event type + for attr in dir(self): + if attr in propnames: + # we have to be careful, not to getattr() on properties which + # may be unset + continue - *args* and *kwargs* are passed to all hooks. - ''' + attr = getattr(self, attr) + if not ishandler(attr): + continue - for hook in system_hooks[event]: - hook(self, *args, **kwargs) + self.add_handler(attr.ha_event, attr) + + + @classmethod + def add_handler(cls, event, handler): + '''Add event handler to subject's class + + :param str event: event identificator + :param collections.Callable handler: handler callable + ''' + + cls.__handlers__[event].add(handler) + + + def fire_event(self, event, *args, **kwargs): + '''Call all handlers for an event + + :param str event: event identificator + + All *args* and *kwargs* are passed verbatim. They are different for + different events. + ''' + + for handler in self.__handlers__[event]: + if hasattr(handler, 'ha_bound'): + # this is our (bound) method, self is implicit + handler(event, *args, **kwargs) + else: + # this is from extension or hand-added, so we see method as + # unbound, therefore we need to pass self + handler(self, event, *args, **kwargs) diff --git a/qubes/ext/__init__.py b/qubes/ext/__init__.py index 01353e78..b9c1e469 100644 --- a/qubes/ext/__init__.py +++ b/qubes/ext/__init__.py @@ -33,22 +33,56 @@ class ExtensionPlugin(qubes.plugins.Plugin): return cls._instance class Extension(object): - '''Base class for all extensions''' + '''Base class for all extensions + + :param qubes.Qubes app: application object + ''' + __metaclass__ = ExtensionPlugin - def __init__(self): + + def __init__(self, app): + self.app = app + for name in dir(self): attr = getattr(self, name) - if not ishook(attr): + if not qubes.events.ishandler(attr): continue - if attr.ho_vm is not None: - attr.ho_vm.add_hook(event, attr) + if attr.ha_vm is not None: + attr.ha_vm.add_hook(attr.ha_event, attr) else: # global hook - qubes.events.add_system_hook(event, attr) + self.app.add_hook(attr.ha_event, attr) + + +def handler(event, vm=None, system=False): + '''Event handler decorator factory. + + To hook an event, decorate a method in your plugin class with this + decorator. You may hook both per-vm-class and global events. + + .. note:: + This decorator is intended only for extensions! For regular use in the + core, see py:func:`qubes.events.handler`. + + :param str event: event type + :param type vm: VM to hook (leave as None to hook all VMs) + :param bool system: when :py:obj:`True`, hook is system-wide (not attached to any VM) + ''' + + def decorator(f): + f.ho_event = event + + if system: + f.ha_vm = None + elif vm is None: + f.ha_vm = qubes.vm.BaseVM + else: + f.ha_vm = vm + + return f + + return decorator -def init(): - for ext in Extension.register.values(): - instance = ext() __all__ = qubes.plugins.load(__file__) diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index 7c7ff150..109197f9 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -12,7 +12,7 @@ Main public classes Helper classes and functions ---------------------------- -.. autoclass:: VMPlugin +.. autoclass:: BaseVMMeta :members: :show-inheritance: @@ -57,19 +57,22 @@ import dateutil.parser import lxml.etree import qubes +import qubes.events import qubes.plugins -class VMPlugin(qubes.plugins.Plugin): +class BaseVMMeta(qubes.plugins.Plugin, qubes.events.EmitterMeta): '''Metaclass for :py:class:`.BaseVM`''' def __init__(cls, name, bases, dict_): - super(VMPlugin, cls).__init__(name, bases, dict_) + super(BaseVMMeta, cls).__init__(name, bases, dict_) cls.__hooks__ = collections.defaultdict(list) -class BaseVM(qubes.PropertyHolder): +class BaseVM(qubes.PropertyHolder, qubes.events.Emitter): '''Base class for all VMs + :param app: Qubes application context + :type app: :py:class:`qubes.Qubes` :param xml: xml node from which to deserialise :type xml: :py:class:`lxml.etree._Element` or :py:obj:`None` @@ -78,23 +81,25 @@ class BaseVM(qubes.PropertyHolder): :py:class:`qubes.vm.qubesvm.QubesVM`. ''' - __metaclass__ = VMPlugin + __metaclass__ = BaseVMMeta - def __init__(self, app, xml=None, load_stage=2, services={}, devices=None, tags={}, **kwargs): + def __init__(self, app, xml, load_stage=2, services={}, devices=None, + tags={}, *args, **kwargs): self.app = app - self.xml = xml self.services = services self.devices = collections.defaultdict(list) if devices is None else devices self.tags = tags all_names = set(prop.__name__ for prop in self.get_props_list(load_stage=2)) - for key in kwargs: + for key in list(kwargs.keys()): if not key in all_names: raise AttributeError( 'No property {!r} found in {!r}'.format( key, self.__class__)) setattr(self, key, kwargs[key]) + del kwargs[key] + super(BaseVM, self).__init__(xml, *args, **kwargs) def add_new_vm(self, vm): '''Add new Virtual Machine to colletion diff --git a/tests/events.py b/tests/events.py new file mode 100644 index 00000000..75d94a93 --- /dev/null +++ b/tests/events.py @@ -0,0 +1,37 @@ +#!/usr/bin/python2 -O + +import sys +import unittest + +sys.path.insert(0, '..') +import qubes.events + +class TC_Emitter(unittest.TestCase): + def test_000_add_handler(self): + # need something mutable + testevent_fired = [False] + + def on_testevent(subject, event): + if event == 'testevent': + testevent_fired[0] = True + + emitter = qubes.events.Emitter() + emitter.add_handler('testevent', on_testevent) + emitter.fire_event('testevent') + self.assertTrue(testevent_fired[0]) + + + def test_001_decorator(self): + class TestEmitter(qubes.events.Emitter): + def __init__(self): + super(TestEmitter, self).__init__() + self.testevent_fired = False + + @qubes.events.handler('testevent') + def on_testevent(self, event): + if event == 'testevent': + self.testevent_fired = True + + emitter = TestEmitter() + emitter.fire_event('testevent') + self.assertTrue(emitter.testevent_fired) diff --git a/tests/init.py b/tests/init.py index 13d4c9c9..936d0812 100644 --- a/tests/init.py +++ b/tests/init.py @@ -104,8 +104,8 @@ class TC_11_VMCollection(unittest.TestCase): # XXX passing None may be wrong in the future self.vms = qubes.VMCollection(None) - self.testvm1 = TestVM(None, qid=1, name='testvm1') - self.testvm2 = TestVM(None, qid=2, name='testvm2') + self.testvm1 = TestVM(None, None, qid=1, name='testvm1') + self.testvm2 = TestVM(None, None, qid=2, name='testvm2') def test_000_contains(self): self.vms._dict = {1: self.testvm1} @@ -132,8 +132,8 @@ class TC_11_VMCollection(unittest.TestCase): with self.assertRaises(TypeError): self.vms.add(object()) - testvm_qid_collision = TestVM(None, name='testvm2', qid=1) - testvm_name_collision = TestVM(None, name='testvm1', qid=2) + testvm_qid_collision = TestVM(None, None, name='testvm2', qid=1) + testvm_name_collision = TestVM(None, None, name='testvm1', qid=2) with self.assertRaises(ValueError): self.vms.add(testvm_qid_collision) From 1a032ecf2aabee8968607bd476eab5b5f3a33334 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Tue, 9 Dec 2014 18:34:00 +0100 Subject: [PATCH 0021/1004] core3: basic global events and their documentation --- doc/conf.py | 12 +++- qubes/__init__.py | 140 +++++++++++++++++++++++++++++++------------ qubes/dochelpers.py | 32 ++++++++++ qubes/events.py | 5 ++ qubes/vm/__init__.py | 7 ++- 5 files changed, 155 insertions(+), 41 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index f000ffcf..fb76e49e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -28,7 +28,17 @@ sys.path.insert(0, os.path.abspath('../')) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'qubes.dochelpers'] +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.viewcode', + + 'qubes.dochelpers', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/qubes/__init__.py b/qubes/__init__.py index 524223b0..792c8549 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -330,6 +330,9 @@ class VMCollection(object): if not isinstance(value, qubes.vm.BaseVM): raise TypeError('{} holds only BaseVM instances'.format(self.__class__.__name__)) + if not hasattr(value, 'qid'): + value.qid = self.domains.get_new_unused_qid() + if value.qid in self: raise ValueError('This collection already holds VM that has qid={!r} (!r)'.format( value.qid, self[value.qid])) @@ -338,6 +341,7 @@ class VMCollection(object): value.name, self[value.name])) self._dict[value.qid] = value + self.app.fire_event('domain-added', value) def __getitem__(self, key): @@ -359,7 +363,9 @@ class VMCollection(object): def __delitem__(self, key): - del self._dict[self[key].qid] + vm = self[key] + del self._dict[vm.qid] + self.app.fire_event('domain-deleted', vm) def __contains__(self, key): @@ -467,12 +473,28 @@ class property(object): def __set__(self, instance, value): + try: + oldvalue = getattr(instance, self.__name__) + has_oldvalue = True + except AttributeError: + has_oldvalue = False + if self._setter is not None: value = self._setter(instance, self, value) if self._type is not None: value = self._type(value) + instance._init_property(self, value) + if has_oldvalue: + instance.fire_event('property-set:' + self.__name__, value, oldvalue) + else: + instance.fire_event('property-set:' + self.__name__, value) + + + def __delete__(self, instance): + delattr(instance, self._attr_name) + def __repr__(self): return '<{} object at {:#x} name={!r} default={!r}>'.format( @@ -505,8 +527,27 @@ class property(object): prop.__name__, self.__class__.__name__)) -class PropertyHolder(object): - '''Abstract class for holding :py:class:`qubes.property`''' +class PropertyHolder(qubes.events.Emitter): + '''Abstract class for holding :py:class:`qubes.property` + + Events fired by instances of this class: + + .. event:: property-load (subject, event) + + Fired once after all properties are loaded from XML. Individual + ``property-set`` events are not fired. + + .. event:: property-set: (subject, event, name, newvalue[, oldvalue]) + + Fired when property changes state. Signature is variable, *oldvalue* is + present only if there was an old value. + + :param name: Property name + :param newvalue: New value of the property + :param oldvalue: Old value of the property + + Members: + ''' def __init__(self, xml, *args, **kwargs): super(PropertyHolder, self).__init__(*args, **kwargs) @@ -545,11 +586,14 @@ class PropertyHolder(object): def load_properties(self, load_stage=None): '''Load properties from immediate children of XML node. + ``property-set`` events are not fired for each individual property. + :param lxml.etree._Element xml: XML node reference ''' # sys.stderr.write('<{}>.load_properties(load_stage={}) xml={!r}\n'.format(hex(id(self)), load_stage, self.xml)) + self.events_enabled = False all_names = set(prop.__name__ for prop in self.get_props_list(load_stage)) # sys.stderr.write(' all_names={!r}\n'.format(all_names)) for node in self.xml.xpath('./properties/property'): @@ -563,6 +607,9 @@ class PropertyHolder(object): name, self.__class__)) setattr(self, name, value) + + self.events_enabled = True + self.fire_event('property-loaded') # sys.stderr.write(' load_properties return\n') @@ -630,22 +677,45 @@ class Qubes(PropertyHolder): :param str store: path to ``qubes.xml`` - The store is loaded in stages. + The store is loaded in stages: - In the first stage there are loaded some basic features from store - (currently labels). + 1. In the first stage there are loaded some basic features from store + (currently labels). - In the second stage stubs for all VMs are loaded. They are filled with - their basic properties, like ``qid`` and ``name``. + 2. In the second stage stubs for all VMs are loaded. They are filled + with their basic properties, like ``qid`` and ``name``. - In the third stage all global properties are loaded. They often reference - VMs, like default netvm, so they should be filled after loading VMs. + 3. In the third stage all global properties are loaded. They often + reference VMs, like default netvm, so they should be filled after + loading VMs. - In the fourth stage all remaining VM properties are loaded. They also need - all VMs loaded, because they represent dependencies between VMs like - aforementioned netvm. + 4. In the fourth stage all remaining VM properties are loaded. They + also need all VMs loaded, because they represent dependencies + between VMs like aforementioned netvm. - In the fifth stage there are some fixups to ensure sane system operation. + 5. In the fifth stage there are some fixups to ensure sane system + operation. + + This class emits following events: + + .. event:: domain-added (subject, event, vm) + + When domain is added. + + :param subject: Event emitter + :param event: Event name (``'domain-added'``) + :param vm: Domain object + + .. event:: domain-deleted (subject, event, vm) + + When domain is deleted. VM still has reference to ``app`` object, + but is not contained within VMCollection. + + :param subject: Event emitter + :param event: Event name (``'domain-deleted'``) + :param vm: Domain object + + Methods and attributes: ''' default_netvm = VMProperty('default_netvm', load_stage=3, @@ -822,22 +892,16 @@ class Qubes(PropertyHolder): return labels + def add_new_vm(self, vm): '''Add new Virtual Machine to colletion ''' - if not hasattr(vm, 'qid'): - vm.qid = self.domains.get_new_unused_qid() - self.domains.add(vm) - # - # XXX - # all this will be moved to an event handler - # and deduplicated with self.load() - # - + @qubes.events.handler('domain-added') + def on_domain_addedd(self, event, vm): # make first created NetVM the default one if not hasattr(self, 'default_fw_netvm') \ and vm.provides_network \ @@ -866,23 +930,21 @@ class Qubes(PropertyHolder): and hasattr(vm, 'netvm'): self.default_clockvm = vm - # XXX don't know if it should return self - return vm - # XXX This was in QubesVmCollection, will be in an event -# def pop(self, qid): -# if self.default_netvm_qid == qid: -# self.default_netvm_qid = None -# if self.default_fw_netvm_qid == qid: -# self.default_fw_netvm_qid = None -# if self.clockvm_qid == qid: -# self.clockvm_qid = None -# if self.updatevm_qid == qid: -# self.updatevm_qid = None -# if self.default_template_qid == qid: -# self.default_template_qid = None -# -# return super(QubesVmCollection, self).pop(qid) + @qubes.events.handler('domain-deleted') + def on_domain_deleted(self, event, vm): + if self.default_netvm == vm: + del self.default_netvm + if self.default_fw_netvm == vm: + del self.default_fw_netvm + if self.clockvm == vm: + del self.clockvm + if self.updatevm == vm: + del self.updatevm + if self.default_template == vm: + del self.default_template + + return super(QubesVmCollection, self).pop(qid) # load plugins diff --git a/qubes/dochelpers.py b/qubes/dochelpers.py index 5d0700bd..4c5a568c 100644 --- a/qubes/dochelpers.py +++ b/qubes/dochelpers.py @@ -10,6 +10,7 @@ particulary our custom Sphinx extension. import csv import posixpath +import re import sys import urllib2 @@ -18,7 +19,9 @@ import docutils.nodes import docutils.parsers.rst import docutils.parsers.rst.roles import docutils.statemachine +import sphinx import sphinx.locale +import sphinx.util.docfields def fetch_ticket_info(uri): '''Fetch info about particular trac ticket given @@ -116,6 +119,30 @@ class VersionCheck(docutils.parsers.rst.Directive): return [node] +# +# this is lifted from sphinx' own conf.py +# + +event_sig_re = re.compile(r'([a-zA-Z-]+)\s*\((.*)\)') + +def parse_event(env, sig, signode): + m = event_sig_re.match(sig) + if not m: + signode += sphinx.addnodes.desc_name(sig, sig) + return sig + name, args = m.groups() + signode += sphinx.addnodes.desc_name(name, name) + plist = sphinx.addnodes.desc_parameterlist() + for arg in args.split(','): + arg = arg.strip() + plist += sphinx.addnodes.desc_parameter(arg, arg) + signode += plist + return name + +# +# end of codelifting +# + def setup(app): app.add_role('ticket', ticket) app.add_config_value('ticket_base_uri', 'https://wiki.qubes-os.org/ticket/', 'env') @@ -124,5 +151,10 @@ def setup(app): man=(visit, depart)) app.add_directive('versioncheck', VersionCheck) + fdesc = sphinx.util.docfields.GroupedField('parameter', label='Parameters', + names=['param'], can_collapse=True) + app.add_object_type('event', 'event', 'pair: %s; event', parse_event, + doc_field_types=[fdesc]) + # vim: ts=4 sw=4 et diff --git a/qubes/events.py b/qubes/events.py index c8cad69b..846b6626 100644 --- a/qubes/events.py +++ b/qubes/events.py @@ -57,6 +57,8 @@ class Emitter(object): def __init__(self, *args, **kwargs): super(Emitter, self).__init__(*args, **kwargs) + self.events_enabled = True + try: propnames = set(prop.__name__ for prop in self.get_props_list()) except AttributeError: @@ -95,6 +97,9 @@ class Emitter(object): different events. ''' + if not self.events_enabled: + return + for handler in self.__handlers__[event]: if hasattr(handler, 'ha_bound'): # this is our (bound) method, self is implicit diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index 109197f9..e6ebbb3a 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -68,7 +68,7 @@ class BaseVMMeta(qubes.plugins.Plugin, qubes.events.EmitterMeta): cls.__hooks__ = collections.defaultdict(list) -class BaseVM(qubes.PropertyHolder, qubes.events.Emitter): +class BaseVM(qubes.PropertyHolder): '''Base class for all VMs :param app: Qubes application context @@ -90,6 +90,7 @@ class BaseVM(qubes.PropertyHolder, qubes.events.Emitter): self.devices = collections.defaultdict(list) if devices is None else devices self.tags = tags + self.events_enabled = False all_names = set(prop.__name__ for prop in self.get_props_list(load_stage=2)) for key in list(kwargs.keys()): if not key in all_names: @@ -101,6 +102,10 @@ class BaseVM(qubes.PropertyHolder, qubes.events.Emitter): super(BaseVM, self).__init__(xml, *args, **kwargs) + self.events_enabled = True + self.fire_event('property-load') + + def add_new_vm(self, vm): '''Add new Virtual Machine to colletion From ef4f00dac0b537c697bd27642957441227468567 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Wed, 17 Dec 2014 13:29:03 +0100 Subject: [PATCH 0022/1004] qubes/vm: DeviceManager class for herding devices collections.defaultdict was not enough, because it cannot pass any arguments to factory. We need to pass domain object and device class to fire events on attach and detach. --- qubes/vm/__init__.py | 69 ++++++++++++++++++++++++++++++++++- tests/vm.py | 85 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 152 insertions(+), 2 deletions(-) diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index e6ebbb3a..19ce16e7 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -68,6 +68,73 @@ class BaseVMMeta(qubes.plugins.Plugin, qubes.events.EmitterMeta): cls.__hooks__ = collections.defaultdict(list) +class DeviceCollection(object): + '''Bag for devices. + + Used as default value for :py:meth:`DeviceManager.__missing__` factory. + + :param vm: VM for which we manage devices + :param class_: device class + ''' + + def __init__(self, vm, class_): + self._vm = vm + self._class = class_ + self._set = set() + + + def attach(self, device): + '''Attach (add) device to domain. + + :param str device: device identifier (format is class-dependent) + ''' + + if device in self: + raise KeyError( + 'device {!r} of class {} already attached to {!r}'.format( + device, self._class, self._vm)) + self._vm.fire_event('device-pre-attached:{}'.format(self._class), device) + self._set.add(device) + self._vm.fire_event('device-attached:{}'.format(self._class), device) + + + def detach(self, device): + '''Detach (remove) device from domain. + + :param str device: device identifier (format is class-dependent) + ''' + + if device not in self: + raise KeyError( + 'device {!r} of class {} not attached to {!r}'.format( + device, self._class, self._vm)) + self._vm.fire_event('device-pre-detached:{}'.format(self._class), device) + self._set.remove(device) + self._vm.fire_event('device-detached:{}'.format(self._class), device) + + + def __iter__(self): + return iter(self._set) + + + def __contains__(self, item): + return item in self._set + + +class DeviceManager(dict): + '''Device manager that hold all devices by their classess. + + :param vm: VM for which we manage devices + ''' + + def __init__(self, vm): + super(DeviceManager, self).__init__() + self._vm = vm + + def __missing__(self, key): + return DeviceCollection(self._vm, key) + + class BaseVM(qubes.PropertyHolder): '''Base class for all VMs @@ -87,7 +154,7 @@ class BaseVM(qubes.PropertyHolder): tags={}, *args, **kwargs): self.app = app self.services = services - self.devices = collections.defaultdict(list) if devices is None else devices + self.devices = DeviceManager(self) if devices is None else devices self.tags = tags self.events_enabled = False diff --git a/tests/vm.py b/tests/vm.py index 3b66a491..f716b652 100644 --- a/tests/vm.py +++ b/tests/vm.py @@ -6,9 +6,92 @@ import unittest import lxml.etree sys.path.insert(0, '../') +import qubes +import qubes.events import qubes.vm +class TestEmitter(qubes.events.Emitter): + def __init__(self): + super(TestEmitter, self).__init__() + self.device_pre_attached_fired = False + self.device_attached_fired = False + self.device_pre_detached_fired = False + self.device_detached_fired = False + + @qubes.events.handler('device-pre-attached:testclass') + def on_device_pre_attached(self, event, dev): + self.device_pre_attached_fired = True + + @qubes.events.handler('device-attached:testclass') + def on_device_attached(self, event, dev): + if self.device_pre_attached_fired: + self.device_attached_fired = True + + @qubes.events.handler('device-pre-detached:testclass') + def on_device_pre_detached(self, event, dev): + if self.device_attached_fired: + self.device_pre_detached_fired = True + + @qubes.events.handler('device-detached:testclass') + def on_device_detached(self, event, dev): + if self.device_pre_detached_fired: + self.device_detached_fired = True + +class TC_00_DeviceCollection(unittest.TestCase): + def setUp(self): + self.emitter = TestEmitter() + self.collection = qubes.vm.DeviceCollection(self.emitter, 'testclass') + + def test_000_init(self): + self.assertFalse(self.collection._set) + + def test_001_attach(self): + self.collection.attach('testdev') + self.assertTrue(self.emitter.device_pre_attached_fired) + self.assertTrue(self.emitter.device_attached_fired) + self.assertFalse(self.emitter.device_pre_detached_fired) + self.assertFalse(self.emitter.device_detached_fired) + + def test_002_detach(self): + self.collection.attach('testdev') + self.collection.detach('testdev') + self.assertTrue(self.emitter.device_pre_attached_fired) + self.assertTrue(self.emitter.device_attached_fired) + self.assertTrue(self.emitter.device_pre_detached_fired) + self.assertTrue(self.emitter.device_detached_fired) + + def test_010_empty_detach(self): + with self.assertRaises(LookupError): + self.collection.detach('testdev') + + def test_011_double_attach(self): + self.collection.attach('testdev') + + with self.assertRaises(LookupError): + self.collection.attach('testdev') + + def test_012_double_detach(self): + self.collection.attach('testdev') + self.collection.detach('testdev') + + with self.assertRaises(LookupError): + self.collection.detach('testdev') + + +class TC_01_DeviceManager(unittest.TestCase): + def setUp(self): + self.emitter = TestEmitter() + self.manager = qubes.vm.DeviceManager(self.emitter) + + def test_000_init(self): + self.assertEqual(self.manager, {}) + + def test_001_missing(self): + self.manager['testclass'].attach('testdev') + self.assertTrue(self.emitter.device_attached_fired) + + class TestVM(qubes.vm.BaseVM): qid = qubes.property('qid', type=int) name = qubes.property('name') @@ -16,7 +99,7 @@ class TestVM(qubes.vm.BaseVM): testlabel = qubes.property('testlabel') defaultprop = qubes.property('defaultprop', default='defaultvalue') -class TC_BaseVM(unittest.TestCase): +class TC_10_BaseVM(unittest.TestCase): def setUp(self): self.xml = lxml.etree.XML(''' From f149c7b59ba9f4139548df80b3a239b0023683cf Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Wed, 17 Dec 2014 13:32:58 +0100 Subject: [PATCH 0023/1004] qubes/vm: fixed __repr__ for BaseVM Previously it could fail with AttributeError when any of the properties was unset. --- qubes/vm/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index 19ce16e7..df3c73af 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -285,11 +285,16 @@ class BaseVM(qubes.PropertyHolder): return element def __repr__(self): - return '<{} object at {:#x} {}>'.format( - self.__class__.__name__, id(self), - ' '.join('{}={}'.format(prop.__name__, getattr(self, prop.__name__)) - for prop in self.get_props_list())) + proprepr = [] + for prop in self.get_props_list(): + try: + proprepr.append('{}={!r}'.format( + prop.__name__, getattr(self, prop.__name__))) + except AttributeError: + continue + return '<{} object at {:#x} {}>'.format( + self.__class__.__name__, id(self), ' '.join(proprepr)) @classmethod def add_hook(cls, event, f): From f9658ae3381c40a57f460ca9dedfc25f2365d7f0 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Wed, 17 Dec 2014 13:35:06 +0100 Subject: [PATCH 0024/1004] qubes/vm: remove old event methods that were overlooked --- qubes/vm/__init__.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index df3c73af..f8cf4e06 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -296,32 +296,6 @@ class BaseVM(qubes.PropertyHolder): return '<{} object at {:#x} {}>'.format( self.__class__.__name__, id(self), ' '.join(proprepr)) - @classmethod - def add_hook(cls, event, f): - '''Add hook to entire VM class and all subclasses - - :param str event: event type - :param callable f: function to fire on event - - Prototype of the function depends on the exact type of event. Classes - which inherit from this class will also inherit the hook. - ''' - - cls.__hooks__[event].append(f) - - - def fire_hooks(self, event, *args, **kwargs): - '''Fire hooks associated with an event - - :param str event: event type - - *args* and *kwargs* are passed to each function - ''' - - for cls in self.__class__.__mro__: - if not hasattr(cls, '__hooks__'): continue - for hook in cls.__hooks__[event]: - hook(self, *args, **kwargs) def load(class_, D): From 6b7860995b5588d4da9d8d3fc7ec11de43fee04d Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Wed, 17 Dec 2014 13:36:58 +0100 Subject: [PATCH 0025/1004] qubes/events.py: Fire events from parent classes too --- qubes/events.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/qubes/events.py b/qubes/events.py index 846b6626..fa5a9d64 100644 --- a/qubes/events.py +++ b/qubes/events.py @@ -16,6 +16,10 @@ def handler(event): To hook an event, decorate a method in your plugin class with this decorator. + It probably makes no sense to specify more than one handler for specific + event in one class, because handlers are not run concurrently and there is + no guarantee of the order of execution. + .. note:: For hooking events from extensions, see :py:func:`qubes.ext.handler`. @@ -89,7 +93,12 @@ class Emitter(object): def fire_event(self, event, *args, **kwargs): - '''Call all handlers for an event + '''Call all handlers for an event. + + Handlers are called for class and all parent classess, in method + resolution order. For each class first are called bound handlers + (specified in class definition), then handlers from extensions. Aside + from above, remaining order is undefined. :param str event: event identificator @@ -100,11 +109,16 @@ class Emitter(object): if not self.events_enabled: return - for handler in self.__handlers__[event]: - if hasattr(handler, 'ha_bound'): - # this is our (bound) method, self is implicit - handler(event, *args, **kwargs) - else: - # this is from extension or hand-added, so we see method as - # unbound, therefore we need to pass self - handler(self, event, *args, **kwargs) + for cls in self.__class__.__mro__: + # first fire bound (= our own) handlers, then handlers from extensions + if not hasattr(cls, '__handlers__'): + continue + for handler in sorted(cls.__handlers__[event], + key=(lambda handler: hasattr(handler, 'ha_bound')), reverse=True): + if hasattr(handler, 'ha_bound'): + # this is our (bound) method, self is implicit + handler(event, *args, **kwargs) + else: + # this is from extension or hand-added, so we see method as + # unbound, therefore we need to pass self + handler(self, event, *args, **kwargs) From 41fef46db2e8cb6aea1d7806c4ec180a32f159c5 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 29 Dec 2014 12:46:16 +0100 Subject: [PATCH 0026/1004] core3 move: QubesVM This is a big commit and probably incomplete. Tests will follow. --- doc/conf.py | 4 + qubes/__init__.py | 142 +++- qubes/config.py | 93 +++ qubes/utils.py | 54 ++ qubes/vm/__init__.py | 296 +++++++ qubes/vm/appvm.py | 9 + qubes/vm/qubesvm.py | 1650 +++++++++++++++++++++++++++++++++++++++- qubes/vm/templatevm.py | 6 +- tests/vm/qubesvm.py | 79 ++ 9 files changed, 2307 insertions(+), 26 deletions(-) create mode 100644 qubes/config.py create mode 100644 qubes/utils.py create mode 100644 tests/vm/qubesvm.py diff --git a/doc/conf.py b/doc/conf.py index fb76e49e..bb05dec8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -33,6 +33,7 @@ extensions = [ 'sphinx.ext.autosummary', 'sphinx.ext.coverage', 'sphinx.ext.doctest', + 'sphinx.ext.graphviz', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.viewcode', @@ -180,6 +181,9 @@ html_last_updated_fmt = '%d.%m.%Y' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None +# html links do not work with svg! +graphviz_output_format = 'png' + # Output file base name for HTML help builder. htmlhelp_basename = 'core-admin-doc' diff --git a/qubes/__init__.py b/qubes/__init__.py index 792c8549..6659c324 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -58,7 +58,7 @@ class QubesException(Exception): '''Exception that can be shown to the user''' pass -class QubesVMMConnection(object): +class VMMConnection(object): '''Connection to Virtual Machine Manager (libvirt)''' def __init__(self): self._libvirt_conn = None @@ -118,13 +118,18 @@ class QubesVMMConnection(object): self.init_vmm_connection() return self._xs -vmm = QubesVMMConnection() - class QubesHost(object): - '''Basic information about host machine''' - def __init__(self): - (model, memory, cpus, mhz, nodes, socket, cores, threads) = vmm.libvirt_conn.getInfo() + '''Basic information about host machine + + :param Qubes app: Qubes application context (must have :py:attr:`Qubes.vmm` attribute defined) + ''' + + def __init__(self, app): + self._app = app + + (model, memory, cpus, mhz, nodes, socket, cores, threads) = \ + self._app.vmm.libvirt_conn.getInfo() self._total_mem = long(memory)*1024 self._no_cpus = cpus @@ -154,7 +159,7 @@ class QubesHost(object): if previous is None: previous_time = time.time() previous = {} - info = vmm.xc.domain_getinfo(0, qubes_max_qid) + info = self._app.vmm.xc.domain_getinfo(0, qubes_max_qid) for vm in info: previous[vm['domid']] = {} previous[vm['domid']]['cpu_time'] = ( @@ -164,7 +169,7 @@ class QubesHost(object): current_time = time.time() current = {} - info = vmm.xc.domain_getinfo(0, qubes_max_qid) + info = self._app.vmm.xc.domain_getinfo(0, qubes_max_qid) for vm in info: current[vm['domid']] = {} current[vm['domid']]['cpu_time'] = ( @@ -427,18 +432,38 @@ class property(object): :param str name: name of the property :param collections.Callable setter: if not :py:obj:`None`, this is used to initialise value; first parameter to the function is holder instance and the second is value; this is called before ``type`` + :param collections.Callable saver: function to coerce value to something readable by setter :param type type: if not :py:obj:`None`, value is coerced to this type - :param object default: default value + :param object default: default value; if callable, will be called with holder as first argument :param int load_stage: stage when property should be loaded (see :py:class:`Qubes` for description of stages) :param int order: order of evaluation (bigger order values are later) :param str doc: docstring; you may use RST markup + Setters and savers have following signatures: + + .. :py:function:: setter(self, prop, value) + :noindex: + + :param self: instance of object that is holding property + :param prop: property object + :param value: value being assigned + + .. :py:function:: saver(self, prop, value) + :noindex: + + :param self: instance of object that is holding property + :param prop: property object + :param value: value being saved + :rtype: str + :raises property.DontSave: when property should not be saved at all + ''' - def __init__(self, name, setter=None, type=None, default=None, + def __init__(self, name, setter=None, saver=None, type=None, default=None, load_stage=2, order=0, save_via_ref=False, doc=None): self.__name__ = name self._setter = setter + self._saver = saver if saver is not None else (lambda self, prop, value: str(value)) self._type = type self._default = default self.order = order @@ -484,6 +509,12 @@ class property(object): if self._type is not None: value = self._type(value) + if has_oldvalue: + instance.fire_event('property-pre-set:' + self.__name__, value, oldvalue) + else: + instance.fire_event('property-pre-set:' + self.__name__, value) + + instance._init_property(self, value) if has_oldvalue: @@ -509,13 +540,28 @@ class property(object): return self.__name__ == other.__name__ + # + # exceptions + # + + class DontSave(Exception): + '''This exception may be raised from saver to sing that property should + not be saved. + ''' + pass + + @staticmethod + def dontsave(self, prop, value): + '''Dummy saver that never saves anything.''' + raise DontSave() + # # some setters provided # @staticmethod def forbidden(self, prop, value): - '''Property setter that forbids loading a property + '''Property setter that forbids loading a property. This is used to effectively disable property in classes which inherit unwanted property. When someone attempts to load such a property, it @@ -527,6 +573,22 @@ class property(object): prop.__name__, self.__class__.__name__)) + @staticmethod + def bool(self, prop, value): + '''Property setter for boolean properties. + + It accepts (case-insensitive) ``'0'``, ``'no'`` and ``false`` as + :py:obj:`False` and ``'1'``, ``'yes'`` and ``'true'`` as + :py:obj:`True`. + ''' + + lcvalue = value.lower() + if lcvalue in ('0', 'no', 'false'): return False + if lcvalue in ('1', 'yes', 'true'): return True + raise ValueError('Invalid literal for boolean property: {!r}'.format(value)) + + + class PropertyHolder(qubes.events.Emitter): '''Abstract class for holding :py:class:`qubes.property` @@ -539,8 +601,17 @@ class PropertyHolder(qubes.events.Emitter): .. event:: property-set: (subject, event, name, newvalue[, oldvalue]) - Fired when property changes state. Signature is variable, *oldvalue* is - present only if there was an old value. + Fired when property changes state. Signature is variable, + *oldvalue* is present only if there was an old value. + + :param name: Property name + :param newvalue: New value of the property + :param oldvalue: Old value of the property + + .. event:: property-pre-set: (subject, event, name, newvalue[, oldvalue]) + + Fired before property changes state. Signature is variable, + *oldvalue* is present only if there was an old value. :param name: Property name :param newvalue: New value of the property @@ -625,11 +696,16 @@ class PropertyHolder(qubes.events.Emitter): for prop in self.get_props_list(): try: - value = str(getattr(self, (prop.__name__ if with_defaults else prop._attr_name))) + value = getattr(self, (prop.__name__ if with_defaults else prop._attr_name)) except AttributeError, e: # sys.stderr.write('AttributeError: {!s}\n'.format(e)) continue + try: + value = prop._saver(self, prop, value) + except property.DontSave: + continue + element = lxml.etree.Element('property', name=prop.__name__) if prop.save_via_ref: element.set('ref', value) @@ -640,8 +716,30 @@ class PropertyHolder(qubes.events.Emitter): return properties -import qubes.vm.qubesvm -import qubes.vm.templatevm + # this was clone_attrs + def clone_properties(self, src, proplist=None): + '''Clone properties from other object. + + :param PropertyHolder src: source object + :param list proplist: list of properties (:py:obj:`None` for all properties) + ''' + + if proplist is None: + proplist = self.get_props_list() + else: + proplist = [prop for prop in self.get_props_list() + if prop.__name__ in proplist or prop in proplist] + + for prop in self.proplist(): + try: + self._init_property(self, prop, getattr(src, prop._attr_name)) + except AttributeError: + continue + + self.fire_event('cloned-properties', src, proplist) + + +import qubes.vm class VMProperty(property): @@ -672,6 +770,10 @@ class VMProperty(property): super(VMProperty, self).__set__(self, instance, vm) +import qubes.vm.qubesvm +import qubes.vm.templatevm + + class Qubes(PropertyHolder): '''Main Qubes application @@ -742,6 +844,12 @@ class Qubes(PropertyHolder): #: collection of all available labels for VMs self.labels = {} + #: Connection to VMM + self.vmm = VMMConnection() + + #: Information about host system + self.host = QubesHost(self) + self._store = store try: @@ -749,7 +857,7 @@ class Qubes(PropertyHolder): except IOError: self._init() - super(PropertyHolder, self).__init__(xml=lxml.etree.parse(self.qubes_store_file)) + super(Qubes, self).__init__(xml=lxml.etree.parse(self.qubes_store_file)) def _open_store(self): diff --git a/qubes/config.py b/qubes/config.py new file mode 100644 index 00000000..7cbaa382 --- /dev/null +++ b/qubes/config.py @@ -0,0 +1,93 @@ +#!/usr/bin/python2 +# -*- coding: utf-8 -*- +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2010 Joanna Rutkowska +# Copyright (C) 2014 Wojtek Porczyk +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# + + +qubes_base_dir = "/var/lib/qubes" +system_path = { + 'qubes_guid_path': '/usr/bin/qubes-guid', + 'qrexec_daemon_path': '/usr/lib/qubes/qrexec-daemon', + 'qrexec_client_path': '/usr/lib/qubes/qrexec-client', + 'qubesdb_daemon_path': '/usr/sbin/qubesdb-daemon', + + 'qubes_base_dir': qubes_base_dir, + + # Relative to qubes_base_dir + 'qubes_appvms_dir': 'appvms', + 'qubes_templates_dir': 'vm-templates', + 'qubes_servicevms_dir': 'servicevms', + 'qubes_store_filename': 'qubes.xml', + 'qubes_kernels_base_dir': 'vm-kernels', + + # qubes_icon_dir is obsolete + # use QIcon.fromTheme() where applicable + 'qubes_icon_dir': '/usr/share/icons/hicolor/128x128/devices', + + 'qrexec_policy_dir': '/etc/qubes-rpc/policy', + + 'config_template_pv': '/usr/share/qubes/vm-template.xml', + + 'qubes_pciback_cmd': '/usr/lib/qubes/unbind-pci-device.sh', + 'prepare_volatile_img_cmd': '/usr/lib/qubes/prepare-volatile-img.sh', + 'monitor_layout_notify_cmd': '/usr/bin/qubes-monitor-layout-notify', +} + +vm_files = { + 'root_img': 'root.img', + 'rootcow_img': 'root-cow.img', + 'volatile_img': 'volatile.img', + 'clean_volatile_img': 'clean-volatile.img.tar', + 'private_img': 'private.img', + 'kernels_subdir': 'kernels', + 'firewall_conf': 'firewall.xml', + 'whitelisted_appmenus': 'whitelisted-appmenus.list', + 'updates_stat_file': 'updates.stat', +} + +defaults = { + 'libvirt_uri': 'xen:///', + 'memory': 400, + 'kernelopts': "nopat", + 'kernelopts_pcidevs': "nopat iommu=soft swiotlb=4096", + + 'dom0_update_check_interval': 6*3600, + + 'private_img_size': 2*1024*1024*1024, + 'root_img_size': 10*1024*1024*1024, + + 'storage_class': None, + + # how long (in sec) to wait for VMs to shutdown, + # before killing them (when used qvm-run with --wait option), + 'shutdown_counter_max': 60, + + 'vm_default_netmask': "255.255.255.0", + + # Set later + 'appvm_label': None, + 'template_label': None, + 'servicevm_label': None, +} + +max_qid = 254 +max_netid = 254 diff --git a/qubes/utils.py b/qubes/utils.py new file mode 100644 index 00000000..5792d588 --- /dev/null +++ b/qubes/utils.py @@ -0,0 +1,54 @@ +#!/usr/bin/python2 -O +# -*- coding: utf-8 -*- + +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2010 Joanna Rutkowska +# Copyright (C) 2013 Marek Marczykowski +# Copyright (C) 2014 Wojtek Porczyk +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + + +# FIXME: should be outside of QubesVM? +def get_timezone(self): + # fc18 + if os.path.islink('/etc/localtime'): + return '/'.join(os.readlink('/etc/localtime').split('/')[-2:]) + # <=fc17 + elif os.path.exists('/etc/sysconfig/clock'): + clock_config = open('/etc/sysconfig/clock', "r") + clock_config_lines = clock_config.readlines() + clock_config.close() + zone_re = re.compile(r'^ZONE="(.*)"') + for line in clock_config_lines: + line_match = zone_re.match(line) + if line_match: + return line_match.group(1) + else: + # last resort way, some applications makes /etc/localtime + # hardlink instead of symlink... + tz_info = os.stat('/etc/localtime') + if not tz_info: + return None + if tz_info.st_nlink > 1: + p = subprocess.Popen(['find', '/usr/share/zoneinfo', + '-inum', str(tz_info.st_ino)], + stdout=subprocess.PIPE) + tz_path = p.communicate()[0].strip() + return tz_path.replace('/usr/share/zoneinfo/', '') + return None + diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index f8cf4e06..3a94a0f9 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -297,6 +297,302 @@ class BaseVM(qubes.PropertyHolder): self.__class__.__name__, id(self), ' '.join(proprepr)) + # + # xml serialising methods + # + + @staticmethod + def xml_net_dev(ip, mac, backend): + '''Return ```` node for libvirt xml. + + This was previously _format_net_dev + + :param str ip: IP address of the frontend + :param str mac: MAC (Ethernet) address of the frontend + :param qubes.vm.QubesVM backend: Backend domain + :rtype: lxml.etree._Element + ''' + + interface = lxml.etree.Element('interface', type='ethernet') + interface.append(lxml.etree.Element('mac', address=mac)) + interface.append(lxml.etree.Element('ip', address=ip)) + interface.append(lxml.etree.Element('domain', name=backend.name)) + + return interface + + + @staticmethod + def xml_pci_dev(address): + '''Return ```` node for libvirt xml. + + This was previously _format_pci_dev + + :param str ip: IP address of the frontend + :param str mac: MAC (Ethernet) address of the frontend + :param qubes.vm.QubesVM backend: Backend domain + :rtype: lxml.etree._Element + ''' + + dev_match = re.match('([0-9a-f]+):([0-9a-f]+)\.([0-9a-f]+)', address) + if not dev_match: + raise QubesException("Invalid PCI device address: %s" % address) + + hostdev = lxml.etree.Element('hostdev', type='pci', managed='yes') + source = lxml.etree.Element('source') + source.append(lxml.etree.Element('address', + bus='0x' + dev_match.group(1), + slot='0x' + dev_match.group(2), + function='0x' + dev_match.group(3))) + hostdev.append(source) + return hostdev + + # + # old libvirt XML + # TODO rewrite it to do proper XML synthesis via lxml.etree + # + + def get_config_params(self): + '''Return parameters for libvirt's XML domain config + + .. deprecated:: 3.0-alpha This will go away. + ''' + + args = {} + args['name'] = self.name + if hasattr(self, 'kernels_dir'): + args['kerneldir'] = self.kernels_dir + args['uuidnode'] = '{!r}'.format(self.uuid) \ + if hasattr(self, 'uuid') else '' + args['vmdir'] = self.dir_path + args['pcidevs'] = ''.join(lxml.etree.tostring(self.xml_pci_dev(dev)) + for dev in self.devices['pci']) + args['maxmem'] = str(self.maxmem) + args['vcpus'] = str(self.vcpus) + args['mem'] = str(max(self.memory, self.maxmem)) + + if 'meminfo-writer' in self.services and not self.services['meminfo-writer']: + # If dynamic memory management disabled, set maxmem=mem + args['maxmem'] = args['mem'] + + if self.netvm is not None: + args['ip'] = self.ip + args['mac'] = self.mac + args['gateway'] = self.netvm.gateway + args['dns1'] = self.netvm.gateway + args['dns2'] = self.secondary_dns + args['netmask'] = self.netmask + args['netdev'] = lxml.etree.tostring(self.xml_net_dev(self.ip, self.mac, self.netvm)) + args['disable_network1'] = ''; + args['disable_network2'] = ''; + else: + args['ip'] = '' + args['mac'] = '' + args['gateway'] = '' + args['dns1'] = '' + args['dns2'] = '' + args['netmask'] = '' + args['netdev'] = '' + args['disable_network1'] = ''; + + args.update(self.storage.get_config_params()) + + if hasattr(self, 'kernelopts'): + args['kernelopts'] = self.kernelopts + if self.debug: + self.log.info("Debug mode: adding 'earlyprintk=xen' to kernel opts") + args['kernelopts'] += ' earlyprintk=xen' + + + def create_config_file(self, file_path=None, prepare_dvm=False): + '''Create libvirt's XML domain config file + + If :py:attr:`qubes.vm.qubesvm.QubesVM.uses_custom_config` is true, this + does nothing. + + :param str file_path: Path to file to create (default: :py:attr:`qubes.vm.qubesvm.QubesVM.conf_file`) + :param bool prepare_dvm: If we are in the process of preparing DisposableVM + ''' + + if file_path is None: + file_path = self.conf_file + if self.uses_custom_config: + conf_appvm = open(file_path, "r") + domain_config = conf_appvm.read() + conf_appvm.close() + return domain_config + + f_conf_template = open(self.config_file_template, 'r') + conf_template = f_conf_template.read() + f_conf_template.close() + + template_params = self.get_config_params() + if prepare_dvm: + template_params['name'] = '%NAME%' + template_params['privatedev'] = '' + template_params['netdev'] = re.sub(r"address='[0-9.]*'", "address='%IP%'", template_params['netdev']) + domain_config = conf_template.format(**template_params) + + # FIXME: This is only for debugging purposes + old_umask = os.umask(002) + try: + conf_appvm = open(file_path, "w") + conf_appvm.write(domain_config) + conf_appvm.close() + except: + # Ignore errors + pass + finally: + os.umask(old_umask) + + return domain_config + + + # + # firewall + # TODO rewrite it, have node under + # and possibly integrate with generic policy framework + # + + def write_firewall_conf(self, conf): + '''Write firewall config file. + ''' + defaults = self.get_firewall_conf() + expiring_rules_present = False + for item in defaults.keys(): + if item not in conf: + conf[item] = defaults[item] + + root = lxml.etree.Element( + "QubesFirewallRules", + policy = "allow" if conf["allow"] else "deny", + dns = "allow" if conf["allowDns"] else "deny", + icmp = "allow" if conf["allowIcmp"] else "deny", + yumProxy = "allow" if conf["allowYumProxy"] else "deny" + ) + + for rule in conf["rules"]: + # For backward compatibility + if "proto" not in rule: + if rule["portBegin"] is not None and rule["portBegin"] > 0: + rule["proto"] = "tcp" + else: + rule["proto"] = "any" + element = lxml.etree.Element( + "rule", + address=rule["address"], + proto=str(rule["proto"]), + ) + if rule["netmask"] is not None and rule["netmask"] != 32: + element.set("netmask", str(rule["netmask"])) + if rule.get("portBegin", None) is not None and \ + rule["portBegin"] > 0: + element.set("port", str(rule["portBegin"])) + if rule.get("portEnd", None) is not None and rule["portEnd"] > 0: + element.set("toport", str(rule["portEnd"])) + if "expire" in rule: + element.set("expire", str(rule["expire"])) + expiring_rules_present = True + + root.append(element) + + tree = lxml.etree.ElementTree(root) + + try: + old_umask = os.umask(002) + with open(self.firewall_conf, 'w') as f: + tree.write(f, encoding="UTF-8", pretty_print=True) + f.close() + os.umask(old_umask) + except EnvironmentError as err: + print >> sys.stderr, "{0}: save error: {1}".format( + os.path.basename(sys.argv[0]), err) + return False + + # Automatically enable/disable 'yum-proxy-setup' service based on allowYumProxy + if conf['allowYumProxy']: + self.services['yum-proxy-setup'] = True + else: + if self.services.has_key('yum-proxy-setup'): + self.services.pop('yum-proxy-setup') + + if expiring_rules_present: + subprocess.call(["sudo", "systemctl", "start", + "qubes-reload-firewall@%s.timer" % self.name]) + + return True + + def has_firewall(self): + return os.path.exists (self.firewall_conf) + + def get_firewall_defaults(self): + return { "rules": list(), "allow": True, "allowDns": True, "allowIcmp": True, "allowYumProxy": False } + + def get_firewall_conf(self): + conf = self.get_firewall_defaults() + + try: + tree = lxml.etree.parse(self.firewall_conf) + root = tree.getroot() + + conf["allow"] = (root.get("policy") == "allow") + conf["allowDns"] = (root.get("dns") == "allow") + conf["allowIcmp"] = (root.get("icmp") == "allow") + conf["allowYumProxy"] = (root.get("yumProxy") == "allow") + + for element in root: + rule = {} + attr_list = ("address", "netmask", "proto", "port", "toport", + "expire") + + for attribute in attr_list: + rule[attribute] = element.get(attribute) + + if rule["netmask"] is not None: + rule["netmask"] = int(rule["netmask"]) + else: + rule["netmask"] = 32 + + if rule["port"] is not None: + rule["portBegin"] = int(rule["port"]) + else: + # backward compatibility + rule["portBegin"] = 0 + + # For backward compatibility + if rule["proto"] is None: + if rule["portBegin"] > 0: + rule["proto"] = "tcp" + else: + rule["proto"] = "any" + + if rule["toport"] is not None: + rule["portEnd"] = int(rule["toport"]) + else: + rule["portEnd"] = None + + if rule["expire"] is not None: + rule["expire"] = int(rule["expire"]) + if rule["expire"] <= int(datetime.datetime.now().strftime( + "%s")): + continue + else: + del(rule["expire"]) + + del(rule["port"]) + del(rule["toport"]) + + conf["rules"].append(rule) + + except EnvironmentError as err: + return conf + except (xml.parsers.expat.ExpatError, + ValueError, LookupError) as err: + print("{0}: load error: {1}".format( + os.path.basename(sys.argv[0]), err)) + return None + + return conf def load(class_, D): cls = BaseVM[class_] diff --git a/qubes/vm/appvm.py b/qubes/vm/appvm.py index 8d58c319..bebf1048 100644 --- a/qubes/vm/appvm.py +++ b/qubes/vm/appvm.py @@ -4,5 +4,14 @@ import qubes.vm.qubesvm class AppVM(qubes.vm.qubesvm.QubesVM): '''Application VM''' + + template = qubes.VMProperty('template', load_stage=4, + vmclass=qubes.vm.templatevm.TemplateVM, + doc='Template, on which this AppVM is based.') + def __init__(self, D): super(AppVM, self).__init__(D) + + # Some additional checks for template based VM + assert self.template + self.template.appvms.add(self) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 99b5d755..9c9bc24d 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -1,22 +1,1660 @@ #!/usr/bin/python2 -O +# -*- coding: utf-8 -*- + +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2010 Joanna Rutkowska +# Copyright (C) 2013 Marek Marczykowski +# Copyright (C) 2014 Wojtek Porczyk +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +import datetime +import lxml.etree +import os +import os.path +import re +import shutil +import subprocess +import sys +import time +import uuid +import xml.parsers.expat +import libvirt +import warnings import qubes +import qubes.config +#import qubes.qdb +#import qubes.qmemman +#import qubes.qmemman_algo +import qubes.utils import qubes.vm +qmemman_present = False +try: + import qubes.qmemman_client + qmemman_present = True +except ImportError: + pass + + +def _setter_qid(self, prop, value): + if not 0 <= value <= qubes.MAX_QID: + raise ValueError( + '{} value must be between 0 and qubes.MAX_QID'.format( + prop.__name__)) + return value + + +def _setter_name(self, prop, value): + if not isinstance(value, basestring): + raise TypeError('{} value must be string, {!r} found'.format( + prop.__name__, type(value).__name__)) + if len(value) > 31: + raise ValueError('{} value must be shorter than 32 characters'.format( + prop.__name__)) + if re.match(r"^[a-zA-Z][a-zA-Z0-9_-]*$", value) is None: + raise ValueError('{} value contains illegal characters'.format( + prop.__name__)) + if self.is_running(): + raise qubes.QubesException('Cannot change name of running VM') + + try: + if self.installed_by_rpm: + raise qubes.QubesException('Cannot rename VM installed by RPM -- ' + 'first clone VM and then use yum to remove package.') + except AttributeError: + pass + + return value + + +def _setter_kernel(self, prop, value): + if not os.path.exists(os.path.join(system_path[ + 'qubes_kernels_base_dir'], value)): + raise qubes.QubesException('Kernel {!r} not installed'.format(value)) + for f in ('vmlinuz', 'modules.img'): + if not os.path.exists(os.path.join( + system_path['qubes_kernels_base_dir'], value, f)): + raise qubes.QubesException( + 'Kernel {!r} not properly installed: missing {!r} file'.format( + value, f)) + return value + + + class QubesVM(qubes.vm.BaseVM): '''Base functionality of Qubes VM shared between all VMs.''' - label = qubes.property('label', - setter=(lambda self, prop, value: self.app.labels[int(value.rsplit('-', 1)[1])]), - doc='Colourful label assigned to VM. This is where you set the colour of the padlock.') + # + # properties loaded from XML + # + label = qubes.property('label', + setter=(lambda self, prop, value: self.app.labels[ + int(value.rsplit('-', 1)[1])]), + doc='Colourful label assigned to VM. This is where you set the colour ' + 'of the padlock.') + + # XXX swallowed uses_default_netvm netvm = qubes.property('netvm', load_stage=4, default=(lambda self: self.app.default_fw_netvm if self.provides_network - else self.app.default_fw_netvm), + else self.app.default_netvm), doc='VM that provides network connection to this domain. ' 'When :py:obj:`False`, machine is disconnected. ' 'When :py:obj:`None` (or absent), domain uses default NetVM.') - provides_network = qubes.property('provides_network', - type=bool, + provides_network = qubes.property('provides_network', type=bool, doc=':py:obj:`True` if it is NetVM or ProxyVM, false otherwise') + + qid = qubes.property('qid', type=int, + setter=_setter_qid, + doc='Internal, persistent identificator of particular domain. ' + 'Note this is different from Xen domid.') + + name = qubes.property('name', type=str, + doc='User-specified name of the domain.') + + uuid = qubes.property('uuid', type=uuid.UUID, default=None, + doc='UUID from libvirt.') + + dir_path = qubes.property('dir_path', type=str, default=None, + doc='FIXME') + + conf_file = qubes.property('conf_file', type=str, + default=(lambda self: self.name + '.conf'), + saver=(lambda self, prop, value: self.relative_path(value)), + doc='libvirt config file?') + + # XXX this should be part of qubes.xml + firewall_conf = qubes.property('firewall_conf', type=str, + default='firewall.xml') + + installed_by_rpm = qubes.property('installed_by_rpm', type=bool, default=False, + setter=qubes.property.bool, + doc="If this domain's image was installed from package tracked by " + "package manager.") + + memory = qubes.property('memory', type=int, default=qubes.config.defaults['memory'], + doc='Memory currently available for this VM.') + + maxmem = qubes.property('maxmem', type=int, default=None, + doc='Maximum amount of memory available for this VM ' + '(for the purpose of memory balancer).') + + internal = qubes.property('internal', type=bool, default=False, + setter=qubes.property.bool, + doc="Internal VM (not shown in qubes-manager, doesn't create appmenus entries.") + + # XXX what is that + vcpus = qubes.property('vcpus', default=None, + doc='FIXME') + + # XXX swallowed uses_default_kernel + # XXX not applicable to HVM? + kernel = qubes.property('kernel', type=str, + setter=_setter_kernel, + default=(lambda self: self.app.default_kernel), + doc='Kernel used by this domain.') + + # XXX swallowed uses_default_kernelopts + # XXX not applicable to HVM? + kernelopts = qubes.property('kernelopts', type=str, load_stage=4, + default=(lambda self: defaults['kernelopts_pcidevs'] \ + if len(self.devices['pci']) > 0 else defaults['kernelopts']), + doc='Kernel command line passed to domain.') + + mac = qubes.property('mac', type=str, + default=(lambda self: '00:16:3E:5E:6C:{:02X}'.format(self.qid)), + doc='MAC address of the NIC emulated inside VM') + + debug = qubes.property('debug', type=bool, default=False, + setter=qubes.property.bool, + doc='Turns on debugging features.') + + # XXX what this exactly does? + # XXX shouldn't this go to standalone VM and TemplateVM, and leave here + # only plain property? + default_user = qubes.property('default_user', type=str, + default=(lambda self: self.template.default_user), + doc='FIXME') + +# @property +# def default_user(self): +# if self.template is not None: +# return self.template.default_user +# else: +# return self._default_user + + qrexec_timeout = qubes.property('qrexec_timeout', type=int, default=60, + doc='Time in seconds after which qrexec connection attempt is deemed failed. ' + 'Operating system inside VM should be able to boot in this time.') + + autostart = qubes.property('autostart', type=bool, default=False, + setter=qubes.property.bool, + doc='Setting this to :py:obj:`True` means that VM should be autostarted on dom0 boot.') + + # XXX I don't understand backups + include_in_backups = qubes.property('include_in_backups', type=bool, default=True, + setter=qubes.property.bool, + doc='If this domain is to be included in default backup.') + + backup_content = qubes.property('backup_content', type=bool, default=False, + setter=qubes.property.bool, + doc='FIXME') + + backup_size = qubes.property('backup_size', type=int, default=0, + doc='FIXME') + + backup_path = qubes.property('backup_path', type=str, default='', + doc='FIXME') + + # format got changed from %s to str(datetime.datetime) + backup_timestamp = qubes.property('backup_timestamp', default=None, + setter=(lambda self, prop, value: datetime.datetime.fromtimestamp(value)), + saver=(lambda self, prop, value: value.strftime('%s')), + doc='FIXME') + + + # + # static, class-wide properties + # + + # config file should go away to storage/backend class + #: template for libvirt config file (XML) + config_file_template = qubes.config.system_path["config_template_pv"] + + # + # properties not loaded from XML, calculated at run-time + # + + # VMM-related + + @property + def xid(self): + '''Xen ID. + + Or not Xen, but ID. + ''' + + if self.libvirt_domain is None: + return -1 + return self.libvirt_domain.ID() + + + @property + def libvirt_domain(self): + '''Libvirt domain object from libvirt. + + May be :py:obj:`None`, if libvirt knows nothing about this domain. + ''' + + if self._libvirt_domain is not None: + return self._libvirt_domain + + try: + if self.uuid is not None: + self._libvirt_domain = vmm.libvirt_conn.lookupByUUID(self.uuid.bytes) + else: + self._libvirt_domain = vmm.libvirt_conn.lookupByName(self.name) + self.uuid = uuid.UUID(bytes=self._libvirt_domain.UUID()) + except libvirt.libvirtError: + if vmm.libvirt_conn.virConnGetLastError()[0] == libvirt.VIR_ERR_NO_DOMAIN: + self._update_libvirt_domain() + else: + raise + return self._libvirt_domain + + + @property + def qdb(self): + '''QubesDB handle for this domain.''' + if self._qdb_connection is None: + if self.is_running(): + self._qdb_connection = qubes.qdb.QubesDB(self.name) + return self._qdb_connection + + + # XXX this should go to to AppVM? + @property + def private_img(self): + '''Location of private image of the VM (that contains :file:`/rw` and :file:`/home`).''' + return self.storage.private_img + + + # XXX this should go to to AppVM? or TemplateVM? + @property + def root_img(self): + '''Location of root image.''' + return self.storage.root_img + + + # XXX and this should go to exactly where? DispVM has it. + @property + def volatile_img(self): + '''Volatile image that overlays :py:attr:`root_img`.''' + return self.storage.volatile_img + + + # XXX shouldn't this go elsewhere? + @property + def updateable(self): + '''True if this machine may be updated on its own.''' + return hasattr(self, 'template') + + + @property + def uses_custom_config(self): + '''True if this machine has config in non-standard place.''' + return self.conf_file != self.absolute_path(self.name + ".conf", None) + + @property + def icon_path(self): + return self.dir_path and os.path.join(self.dir_path, "icon.png") + + + # XXX I don't know what to do with these; probably should be isinstance(...) +# def is_template(self): +# return False +# +# def is_appvm(self): +# return False +# +# def is_proxyvm(self): +# return False +# +# def is_disposablevm(self): +# return False + + + # network-related + + @property + def ip(self): + '''IP address of this domain.''' + if self.netvm is not None: + return self.netvm.get_ip_for_vm(self.qid) + else: + return None + + @property + def netmask(self): + '''Netmask for this domain's IP address.''' + if self.netvm is not None: + return self.netvm.netmask + else: + return None + + @property + def gateway(self): + '''Gateway for other domains that use this domain as netvm.''' + # This is gateway IP for _other_ VMs, so make sense only in NetVMs + return None + + @property + def secondary_dns(self): + '''Secondary DNS server set up for this domain.''' + if self.netvm is not None: + return self.netvm.secondary_dns + else: + return None + + @property + def vif(self): + '''Name of the network interface backend in netvm that is connected to + NIC inside this domain.''' + if self.xid < 0: + return None + if self.netvm is None: + return None + return "vif{0}.+".format(self.xid) + + # + # constructor + # + + def __init__(self, app, xml): + super(QubesVM, self).__init__(app, xml) + + #Init private attrs + + self._libvirt_domain = None + self._qdb_connection = None + + assert self.__qid < qubes_max_qid, "VM id out of bounds!" + assert self.name is not None + + if not self.verify_name(self.name): + msg = ("'%s' is invalid VM name (invalid characters, over 31 chars long, " + "or one of 'none', 'true', 'false')") % self.name + if xml is not None: + print >>sys.stderr, "WARNING: %s" % msg + else: + raise QubesException(msg) + + # Not in generic way to not create QubesHost() to frequently + # XXX this doesn't apply, host is instantiated once + if self.maxmem is None and not self.app.vmm.offline_mode: + total_mem_mb = self.app.host.memory_total/1024 + self.maxmem = total_mem_mb/2 + + # Linux specific cap: max memory can't scale beyond 10.79*init_mem + # XXX what?! -woju + if self.maxmem > self.memory * 10: + self.maxmem = self.memory * 10 + + # By default allow use all VCPUs + if not hasattr(self, 'vcpus') and not self.app.vmm.offline_mode: + self.vcpus = self.app.host.no_cpus + + # Always set if meminfo-writer should be active or not + if 'meminfo-writer' not in self.services: + self.services['meminfo-writer'] = not (len(self.devices['pci']) > 0) + + # Additionally force meminfo-writer disabled when VM have PCI devices + if len(self.devices['pci']) > 0: + self.services['meminfo-writer'] = False + + # Initialize VM image storage class + self.storage = qubes.config.defaults["storage_class"](self) + if hasattr(self, 'kernels_dir'): + self.storage.modules_img = os.path.join(self.kernels_dir, + "modules.img") + self.storage.modules_img_rw = self.kernel is None + + # fire hooks + self.fire_event('domain-init') + + + # + # event handlers + # + + + @qubes.events.handler('property-set:label') + def on_property_set_label(self, event, name, new_label, old_label=None): + if self.icon_path: + try: + os.remove(self.icon_path) + except: + pass + if hasattr(os, "symlink"): + os.symlink(new_label.icon_path, self.icon_path) + # FIXME: some os-independent wrapper? + subprocess.call(['sudo', 'xdg-icon-resource', 'forceupdate']) + else: + shutil.copy(new_label.icon_path, self.icon_path) + + + @qubes.events.handler('property-del:netvm') + def on_property_del_netvm(self, event, name, old_netvm): + # we are changing to default netvm + new_netvm = self.netvm + if new_netvm == old_netvm: return + self.on_property_set_netvm(self, event, name, new_netvm, old_netvm) + + + @qubes.events.handler('property-set:netvm') + def on_property_set_netvm(self, event, name, new_netvm, old_netvm=None): + if self.is_running() and new_netvm is not None and not new_netvm.is_running(): + raise QubesException("Cannot dynamically attach to stopped NetVM") + + if self.netvm is not None: + del self.netvm.connected_vms[self] + if self.is_running(): + self.detach_network() + + # TODO change to domain-removed event handler in netvm +# if hasattr(self.netvm, 'post_vm_net_detach'): +# self.netvm.post_vm_net_detach(self) + + if new_netvm is None: + if not self._do_not_reset_firewall: + # Set also firewall to block all traffic as discussed in #370 + if os.path.exists(self.firewall_conf): + shutil.copy(self.firewall_conf, os.path.join(system_path["qubes_base_dir"], + "backup", "%s-firewall-%s.xml" % (self.name, + time.strftime('%Y-%m-%d-%H:%M:%S')))) + self.write_firewall_conf({'allow': False, 'allowDns': False, + 'allowIcmp': False, 'allowYumProxy': False, 'rules': []}) + else: + new_netvm.connected_vms.add(self) + + if new_netvm is None: + return + + if self.is_running(): + # refresh IP, DNS etc + self.create_qdb_entries() + self.attach_network() + + # TODO domain-added event handler in netvm +# if hasattr(self.netvm, 'post_vm_net_attach'): +# self.netvm.post_vm_net_attach(self) + + + @qubes.events.handler('property-set:name') + def on_property_set_name(self, event, name, new_name, old_name=None): + if self.libvirt_domain: + self.libvirt_domain.undefine() + self._libvirt_domain = None + if self._qdb_connection: + self._qdb_connection.close() + self._qdb_connection = None + + new_conf = os.path.join(self.dir_path, new_name + '.conf') + if os.path.exists(self.conf_file): + os.rename(self.conf_file, new_conf) + + old_dirpath = self.dir_path + + self.storage.rename(self.name, name) + new_dirpath = self.storage.vmdir + self.dir_path = new_dirpath + + if self.conf_file is not None: + self.conf_file = new_conf.replace(old_dirpath, new_dirpath) + if hasattr(self, 'kernels_dir') and self.kernels_dir is not None: + self.kernels_dir = self.kernels_dir.replace(old_dirpath, new_dirpath) + + self._update_libvirt_domain() + self.post_rename(old_name) + + + @qubes.events.handler('property-pre-set:autostart') + def on_property_pre_set_autostart(self, event, prop, name, value, oldvalue=None): + if subprocess.call(['sudo', 'systemctl', + ('enable' if value else 'disable'), + 'qubes-vm@{}.service'.format(self.name)]): + raise QubesException("Failed to set autostart for VM via systemctl") + + + @qubes.events.handler('device-pre-attached:pci') + def on_device_pre_attached_pci(self, event, pci): + if not os.path.exists('/sys/bus/pci/devices/0000:%s' % pci): + raise QubesException("Invalid PCI device: %s" % pci) + if not self.is_running(): + return + + try: + subprocess.check_call(['sudo', system_path["qubes_pciback_cmd"], pci]) + subprocess.check_call(['sudo', 'xl', 'pci-attach', str(self.xid), pci]) + except Exception as e: + print >>sys.stderr, "Failed to attach PCI device on the fly " \ + "(%s), changes will be seen after VM restart" % str(e) + + + @qubes.events.handler('device-pre-detached:pci') + def on_device_pre_detached_pci(self, event, pci): + if not self.is_running(): + return + + p = subprocess.Popen(['xl', 'pci-list', str(self.xid)], + stdout=subprocess.PIPE) + result = p.communicate() + m = re.search(r"^(\d+.\d+)\s+0000:%s$" % pci, result[0], flags=re.MULTILINE) + if not m: + print >>sys.stderr, "Device %s already detached" % pci + return + vmdev = m.group(1) + try: + self.run_service("qubes.DetachPciDevice", + user="root", input="00:%s" % vmdev) + subprocess.check_call(['sudo', 'xl', 'pci-detach', str(self.xid), pci]) + except Exception as e: + print >>sys.stderr, "Failed to detach PCI device on the fly " \ + "(%s), changes will be seen after VM restart" % str(e) + + + # + # methods for changing domain state + # + + def start(self, preparing_dvm=False, start_guid=True, + notify_function=None, mem_required=None): + '''Start domain + + :param bool preparing_dvm: FIXME + :param bool start_guid: FIXME + :param collections.Callable notify_function: FIXME + :param int mem_required: FIXME + ''' + + # Intentionally not used is_running(): eliminate also "Paused", "Crashed", "Halting" + if self.get_power_state() != "Halted": + raise QubesException("VM is already running!") + + self.log.info('Starting {}'.format(self.name)) + + self.verify_files() + + if self.netvm is not None: + if self.netvm.qid != 0: + if not self.netvm.is_running(): + self.netvm.start(start_guid=start_guid, notify_function=notify_function) + + self.storage.prepare_for_vm_startup() + self._update_libvirt_domain() + + if mem_required is None: + mem_required = int(self.memory) * 1024 * 1024 + if qmemman_present: + qmemman_client = qubes.qmemman_client.QMemmanClient() + try: + got_memory = qmemman_client.request_memory(mem_required) + except IOError as e: + raise IOError("ERROR: Failed to connect to qmemman: %s" % str(e)) + if not got_memory: + qmemman_client.close() + raise MemoryError("ERROR: insufficient memory to start VM '%s'" % self.name) + + # Bind pci devices to pciback driver + for pci in self.devices['pci']: + nd = vmm.libvirt_conn.nodeDeviceLookupByName('pci_0000_' + pci.replace(':','_').replace('.','_')) + try: + nd.dettach() + except libvirt.libvirtError: + if vmm.libvirt_conn.virConnGetLastError()[0] == libvirt.VIR_ERR_INTERNAL_ERROR: + # allready detached + pass + else: + raise + + self.libvirt_domain.createWithFlags(libvirt.VIR_DOMAIN_START_PAUSED) + + if preparing_dvm: + self.services['qubes-dvm'] = True + + self.log.info('Setting Qubes DB info for the VM') + self.start_qubesdb() + self.create_qdb_entries() + + self.log.info('Updating firewall rules') + + for vm in self.app.domains: + if vm.is_proxyvm() and vm.is_running(): + vm.write_iptables_xenstore_entry() + + self.fire_event('domain-started', preparing_dvm=preparing_dvm, start_guid=start_guid) + + + self.log.warning('Activating the {} VM'.format(self.name)) + self.libvirt_domain.resume() + + # close() is not really needed, because the descriptor is close-on-exec + # anyway, the reason to postpone close() is that possibly xl is not done + # constructing the domain after its main process exits + # so we close() when we know the domain is up + # the successful unpause is some indicator of it + if qmemman_present: + qmemman_client.close() + + if self._start_guid_first and start_guid and not preparing_dvm and os.path.exists('/var/run/shm.id'): + self.start_guid(notify_function=notify_function) + + if not preparing_dvm: + self.start_qrexec_daemon(notify_function=notify_function) + + if start_guid and not preparing_dvm and os.path.exists('/var/run/shm.id'): + self.start_guid(notify_function=notify_function) + + + def shutdown(self): + '''Shutdown domain. + + :raises QubesException: when domain is already shut down. + ''' + + if not self.is_running(): + raise QubesException("VM already stopped!") + + self.libvirt_domain.shutdown() + + + def force_shutdown(self): + '''Forcefuly shutdown (destroy) domain. + + :raises QubesException: when domain is already shut down. + ''' + + if not self.is_running() and not self.is_paused(): + raise QubesException("VM already stopped!") + + self.libvirt_domain.destroy() + + + def suspend(self): + '''Suspend (pause) domain. + + :raises QubesException: when domain is already shut down. + :raises NotImplemetedError: when domain has PCI devices attached. + ''' + + if not self.is_running() and not self.is_paused(): + raise QubesException("VM already stopped!") + + if len(self.devices['pci']) > 0: + raise NotImplementedError() + else: + self.libvirt_domain.suspend() + + + def pause(self): + '''Pause (suspend) domain. This currently delegates to :py:meth:`suspend`.''' + + if not self.is_running(): + raise QubesException("VM not running!") + + self.suspend() + + + def resume(self): + '''Resume suspended domain. + + :raises NotImplemetedError: when machine is alread suspended. + ''' + + if self.get_power_state() == "Suspended": + raise NotImplementedError() + else: + self.unpause() + + def unpause(self): + '''Resume (unpause) a domain''' + if not self.is_paused(): + raise QubesException("VM not paused!") + + self.libvirt_domain.resume() + + + def run(self, command, user=None, autostart=False, notify_function=None, + passio=False, passio_popen=False, passio_stderr=False, + ignore_stderr=False, localcmd=None, wait=False, gui=True, + filter_esc=False): + '''Run specified command inside domain + + :param str command: the command to be run + :param str user: user to run the command as + :param bool autostart: if :py:obj:`True`, machine will be started if it is not running + :param collections.Callable notify_function: FIXME, may go away + :param bool passio: FIXME + :param bool passio_popen: if :py:obj:`True`, :py:class:`subprocess.Popen` object has connected ``stdin`` and ``stdout`` + :param bool passio_stderr: if :py:obj:`True`, :py:class:`subprocess.Popen` has additionaly ``stderr`` connected + :param bool ignore_stderr: if :py:obj:`True`, ``stderr`` is connected to :file:`/dev/null` + :param str localcmd: local command to communicate with remote command + :param bool wait: if :py:obj:`True`, wait for command completion + :param bool gui: when autostarting, also start gui daemon + :param bool filter_esc: filter escape sequences to protect terminal emulator + ''' + + if user is None: + user = self.default_user + null = None + if not self.is_running() and not self.is_paused(): + if not autostart: + raise QubesException("VM not running") + + try: + if notify_function is not None: + notify_function("info", "Starting the '{0}' VM...".format(self.name)) + self.start(start_guid=gui, notify_function=notify_function) + + except (IOError, OSError, QubesException) as err: + raise QubesException("Error while starting the '{0}' VM: {1}".format(self.name, err)) + except (MemoryError) as err: + raise QubesException("Not enough memory to start '{0}' VM! " + "Close one or more running VMs and try " + "again.".format(self.name)) + + if self.is_paused(): + raise QubesException("VM is paused") + if not self.is_qrexec_running(): + raise QubesException( + "Domain '{}': qrexec not connected.".format(self.name)) + + if gui and os.getenv("DISPLAY") is not None and not self.is_guid_running(): + self.start_guid(verbose = verbose, notify_function = notify_function) + + args = [system_path["qrexec_client_path"], "-d", str(self.name), "%s:%s" % (user, command)] + if localcmd is not None: + args += [ "-l", localcmd] + if filter_esc: + args += ["-t"] + if os.isatty(sys.stderr.fileno()): + args += ["-T"] + if passio: + if os.name == 'nt': + # wait for qrexec-client to exit, otherwise client is not properly attached to console + # if qvm-run is executed from cmd.exe + ret = subprocess.call(args) + exit(ret) + os.execv(system_path["qrexec_client_path"], args) + exit(1) + + call_kwargs = {} + if ignore_stderr: + null = open("/dev/null", "w") + call_kwargs['stderr'] = null + + if passio_popen: + popen_kwargs={'stdout': subprocess.PIPE} + popen_kwargs['stdin'] = subprocess.PIPE + if passio_stderr: + popen_kwargs['stderr'] = subprocess.PIPE + else: + popen_kwargs['stderr'] = call_kwargs.get('stderr', None) + p = subprocess.Popen(args, **popen_kwargs) + if null: + null.close() + return p + if not wait: + args += ["-e"] + retcode = subprocess.call(args, **call_kwargs) + if null: + null.close() + return retcode + + + def run_service(self, service, source=None, user=None, + passio_popen=False, input=None): + '''Run service on this VM + + **passio_popen** and **input** are mutually exclusive. + + :param str service: service name + :param qubes.vm.QubesVM: source domain as presented to this VM + :param str user: username to run service as + :param bool passio_popen: passed verbatim to :py:meth:`run` + :param str input: string passed as input to service + ''' + + if input is not None and passio_popen is not None: + raise ValueError("'input' and 'passio_popen' cannot be used " + "together") + + source = 'dom0' if source is None else self.app.domains[source].name + + if input: + return self.run("QUBESRPC %s %s" % (service, source), + localcmd="echo %s" % input, user=user, wait=True) + else: + return self.run("QUBESRPC %s %s" % (service, source), + passio_popen=passio_popen, user=user, wait=True) + + + + def start_guid(self, extra_guid_args=[]): + '''Launch gui daemon. + + GUI daemon securely displays windows from domain. + + :param list extra_guid_args: Extra argv to pass to :program:`guid`. + ''' + + self.log.info('Starting gui daemon') + + guid_cmd = [system_path["qubes_guid_path"], + "-d", str(self.xid), "-N", self.name, + "-c", self.label.color, + "-i", self.label.icon_path, + "-l", str(self.label.index)] + guid_cmd += extra_guid_args + + if self.debug: + guid_cmd += ['-v', '-v'] + +# elif not verbose: + guid_cmd += ['-q'] + + retcode = subprocess.call(guid_cmd) + if (retcode != 0) : + raise QubesException("Cannot start qubes-guid!") + + self.log.info('Sending monitor layout') + + try: + subprocess.call([system_path["monitor_layout_notify_cmd"], self.name]) + except Exception as e: + self.log.error('ERROR: {!s}'.format(e)) + + self.wait_for_session() + + + def start_qrexec_daemon(self): + '''Start qrexec daemon. + + :raises OSError: when starting fails. + ''' + + self.log.debug('Starting the qrexec daemon') + qrexec_args = [str(self.xid), self.name, self.default_user] + if not self.debug: + qrexec_args.insert(0, "-q") + qrexec_env = os.environ + qrexec_env['QREXEC_STARTUP_TIMEOUT'] = str(self.qrexec_timeout) + retcode = subprocess.call([system_path["qrexec_daemon_path"]] + + qrexec_args, env=qrexec_env) + if (retcode != 0) : + raise OSError("Cannot execute qrexec-daemon!") + + + def start_qubesdb(self): + '''Start QubesDB daemon. + + :raises OSError: when starting fails. + ''' + + self.log.info('Starting Qubes DB') + + retcode = subprocess.call([ + system_path["qubesdb_daemon_path"], + str(self.xid), + self.name]) + if retcode != 0: + self.force_shutdown() + raise OSError("ERROR: Cannot execute qubesdb-daemon!") + + + def wait_for_session(self): + '''Wait until machine finished boot sequence. + + This is done by executing qubes RPC call that checks if dummy system + service (which is started late in standard runlevel) is active. + ''' + + self.log.info('Waiting for qubes-session') + + # Note : User root is redefined to SYSTEM in the Windows agent code + p = self.run('QUBESRPC qubes.WaitForSession none', + user="root", passio_popen=True, gui=False, wait=True) + p.communicate(input=self.default_user) + + + def create_on_disk(self, source_template=None): + '''Create files needed for VM. + + :param qubes.vm.templatevm.TemplateVM source_template: Template to use + (if :py:obj:`None`, use domain's own template + ''' + + if source_template is None: + source_template = self.template + assert source_template is not None + + self.storage.create_on_disk(verbose, source_template) + + if self.updateable: + kernels_dir = source_template.kernels_dir + self.log.info( + 'Copying the kernel (unset kernel to use it): {0}'.format( + kernels_dir)) + + os.mkdir(self.dir_path + '/kernels') + for f in ("vmlinuz", "initramfs", "modules.img"): + shutil.copy(os.path.join(kernels_dir, f), + os.path.join(self.dir_path, + qubes.config.vm_files["kernels_subdir"], f)) + + self.log.info('Creating icon symlink: {0} -> {1}'.format(self.icon_path, self.label.icon_path)) + if hasattr(os, "symlink"): + os.symlink(self.label.icon_path, self.icon_path) + else: + shutil.copy(self.label.icon_path, self.icon_path) + + # fire hooks + self.fire_event('domain-created-on-disk', source_template) + + + def resize_private_img(self, size): + '''Resize private image.''' + + # TODO QubesValueError, not assert + assert size >= self.get_private_img_sz(), "Cannot shrink private.img" + + # resize the image + self.storage.resize_private_img(size) + + # and then the filesystem + retcode = 0 + if self.is_running(): + retcode = self.run("while [ \"`blockdev --getsize64 /dev/xvdb`\" -lt {0} ]; do ".format(size) + + "head /dev/xvdb > /dev/null; sleep 0.2; done; resize2fs /dev/xvdb", user="root", wait=True) + if retcode != 0: + raise QubesException("resize2fs failed") + + + def remove_from_disk(self): + '''Remove domain remnants from disk.''' + self.fire_event('domain-removed-from-disk') + self.storage.remove_from_disk() + + + def clone_disk_files(self, src): + '''Clone files from other vm. + + :param qubes.vm.QubesVM src: source VM + ''' + + if src_vm.is_running(): + raise QubesException("Attempt to clone a running VM!") + + self.storage.clone_disk_files(src, verbose=False) + + if srv.icon_path is not None \ + and os.path.exists(src.dir_path) \ + and self.icon_path is not None: + if os.path.islink(src.icon_path): + icon_path = os.readlink(src.icon_path) + self.log.info( + 'Creating icon symlink {} -> {}'.format( + self.icon_path, icon_path)) + os.symlink(icon_path, self.icon_path) + else: + self.log.info( + 'Copying icon {} -> {}'.format( + src.icon_path, self.icon_path)) + shutil.copy(src.icon_path, self.icon_path) + + # fire hooks + self.fire_event('cloned-files', src) + + + # TODO maybe this should be other way: backend.devices['net'].attach(self) + def attach_network(self): + '''Attach network in this machine to it's netvm.''' + + if not self.is_running(): + raise QubesException("VM not running!") + + if self.netvm is None: + raise QubesException("NetVM not set!") + + if not netvm.is_running(): + self.log.info('Starting NetVM ({0})'.format(netvm.name)) + netvm.start() + + self.libvirt_domain.attachDevice(lxml.etree.ElementTree( + self.xml_net_dev(self.ip, self.mac, self.netvm)).tostring()) + + + def detach_network(self): + '''Detach machine from it's netvm''' + + if not self.is_running(): + raise QubesException("VM not running!") + + if self.netvm is None: + raise QubesException("NetVM not set!") + + + self.libvirt_domain.detachDevice(lxml.etree.ElementTree( + self.xml_net_dev(self.ip, self.mac, self.netvm)).tostring()) + + + # + # methods for querying domain state + # + + # state of the machine + + def get_power_state(self): + '''Return power state description string. + + Return value may be one of those: + + =============== ======================================================== + return value meaning + =============== ======================================================== + ``'Halted'`` Machine is not active. + ``'Transient'`` Machine is running, but does not have :program:`guid` + or :program:`qrexec` available. + ``'Running'`` Machine is ready and running. + ``'Paused'`` Machine is paused (currently not available, see below). + ``'Suspended'`` Machine is S3-suspended. + ``'Halting'`` Machine is in process of shutting down. + ``'Dying'`` Machine crashed and is unusable. + ``'Crashed'`` Machine crashed and is unusable, probably because of + bug in dom0. + ``'NA'`` Machine is in unknown state (most likely libvirt domain + is undefined). + =============== ======================================================== + + ``Paused`` state is currently unavailable because of missing code in libvirt/xen glue. + + FIXME: graph below may be incomplete and wrong. Click on method name to + see its documentation. + + .. graphviz:: + + digraph { + node [fontname="sans-serif"]; + edge [fontname="mono"]; + + + Halted; + NA; + Dying; + Crashed; + Transient; + Halting; + Running; + Paused [color=gray75 fontcolor=gray75]; + Suspended; + + NA -> Halted; + Halted -> NA [constraint=false]; + + Halted -> Transient + [xlabel="start()" URL="#qubes.vm.qubesvm.QubesVM.start"]; + Transient -> Running; + + Running -> Halting + [xlabel="shutdown()" + URL="#qubes.vm.qubesvm.QubesVM.shutdown" + constraint=false]; + Halting -> Dying -> Halted [constraint=false]; + + /* cosmetic, invisible edges to put rank constraint */ + Dying -> Halting [style="invis"]; + Halting -> Transient [style="invis"]; + + Running -> Halted + [label="force_shutdown()" + URL="#qubes.vm.qubesvm.QubesVM.force_shutdown" + constraint=false]; + + Running -> Crashed [constraint=false]; + Crashed -> Halted [constraint=false]; + + Running -> Paused + [label="pause()" URL="#qubes.vm.qubesvm.QubesVM.pause" + color=gray75 fontcolor=gray75]; + Running -> Suspended + [label="pause()" URL="#qubes.vm.qubesvm.QubesVM.pause" + color=gray50 fontcolor=gray50]; + Paused -> Running + [label="unpause()" URL="#qubes.vm.qubesvm.QubesVM.unpause" + color=gray75 fontcolor=gray75]; + Suspended -> Running + [label="unpause()" URL="#qubes.vm.qubesvm.QubesVM.unpause" + color=gray50 fontcolor=gray50]; + + Running -> Suspended + [label="suspend()" URL="#qubes.vm.qubesvm.QubesVM.suspend"]; + Suspended -> Running + [label="resume()" URL="#qubes.vm.qubesvm.QubesVM.resume"]; + + + { rank=source; Halted NA }; + { rank=same; Transient Halting }; + { rank=same; Crashed Dying }; + { rank=sink; Paused Suspended }; + } + + .. seealso:: http://wiki.libvirt.org/page/VM_lifecycle + + .. seealso:: https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainState + + ''' + + libvirt_domain = self.libvirt_domain + if libvirt_domain is None: + return "NA" + + if libvirt_domain.isActive(): + if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_PAUSED: + return "Paused" + elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_CRASHED: + return "Crashed" + elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTDOWN: + return "Halting" + elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTOFF: + return "Dying" + elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_PMSUSPENDED: + return "Suspended" + else: + if not self.is_fully_usable(): + return "Transient" + else: + return "Running" + else: + return 'Halted' + + return "NA" + + + def is_running(self): + '''Check whether this domain is running. + + :returns: :py:obj:`True` if this domain is started, :py:obj:`False` otherwise. + :rtype: bool + ''' + + return self.libvirt_domain and self.libvirt_domain.isActive() + + + def is_paused(self): + '''Check whether this domain is paused. + + :returns: :py:obj:`True` if this domain is paused, :py:obj:`False` otherwise. + :rtype: bool + ''' + + return self.libvirt_domain \ + and self.libvirt_domain.state() == libvirt.VIR_DOMAIN_PAUSED + + + def is_guid_running(self): + '''Check whether gui daemon for this domain is available. + + :returns: :py:obj:`True` if guid is running, :py:obj:`False` otherwise. + :rtype: bool + ''' + xid = self.xid + if xid < 0: + return False + if not os.path.exists('/var/run/qubes/guid-running.%d' % xid): + return False + return True + + + def is_qrexec_running(self): + '''Check whether qrexec for this domain is available. + + :returns: :py:obj:`True` if qrexec is running, :py:obj:`False` otherwise. + :rtype: bool + ''' + if self.xid < 0: + return False + return os.path.exists('/var/run/qubes/qrexec.%s' % self.name) + + + def is_fully_usable(self): + '''Check whether domain is running and sane. + + Currently this checks for running guid and qrexec. + + :returns: :py:obj:`True` if qrexec is running, :py:obj:`False` otherwise. + :rtype: bool + ''' + + # Running gui-daemon implies also VM running + if not self.is_guid_running(): + return False + if not self.is_qrexec_running(): + return False + return True + + + # memory and disk + + def get_mem(self): + '''Get current memory usage from VM. + + :returns: Memory usage [FIXME unit]. + :rtype: FIXME + ''' + + if self.libvirt_domain is None: + return 0 + if not self.libvirt_domain.isActive(): + return 0 + + return self.libvirt_domain.info()[1] + + + def get_mem_static_max(self): + '''Get maximum memory available to VM. + + :returns: Memory limit [FIXME unit]. + :rtype: FIXME + ''' + + if self.libvirt_domain is None: + return 0 + + return self.libvirt_domain.maxMemory() + + + def get_per_cpu_time(self): + '''Get total CPU time burned by this domain since start. + + :returns: CPU time usage [FIXME unit]. + :rtype: FIXME + ''' + + if self.libvirt_domain is None: + return 0 + if not self.libvirt_domain.isActive(): + return 0 + + return libvirt_domain.getCPUStats( + libvirt.VIR_NODE_CPU_STATS_ALL_CPUS, 0)[0]['cpu_time']/10**9 + + + # XXX shouldn't this go only to vms that have root image? + def get_disk_utilization_root_img(self): + '''Get space that is actually ocuppied by :py:attr:`root_img`. + + Root image is a sparse file, so it is probably much less than logical + available space. + + :returns: domain's real disk image size [FIXME unit] + :rtype: FIXME + + .. seealso:: :py:meth:`get_root_img_sz` + ''' + + return qubes.utils.get_disk_usage(self.root_img) + + + # XXX shouldn't this go only to vms that have root image? + def get_root_img_sz(self): + '''Get image size of :py:attr:`root_img`. + + Root image is a sparse file, so it is probably much more than ocuppied + physical space. + + :returns: domain's virtual disk size [FIXME unit] + :rtype: FIXME + + .. seealso:: :py:meth:`get_disk_utilization_root_img` + ''' + + if not os.path.exists(self.root_img): + return 0 + + return os.path.getsize(self.root_img) + + + def get_disk_utilization_private_img(self): + '''Get space that is actually ocuppied by :py:attr:`private_img`. + + Private image is a sparse file, so it is probably much less than + logical available space. + + :returns: domain's real disk image size [FIXME unit] + :rtype: FIXME + + .. seealso:: :py:meth:`get_private_img_sz` + ''' + + return qubes.utils.get_disk_usage(self.private_img) + + + def get_private_img_sz(self): + '''Get image size of :py:attr:`private_img`. + + Private image is a sparse file, so it is probably much more than + ocuppied physical space. + + :returns: domain's virtual disk size [FIXME unit] + :rtype: FIXME + + .. seealso:: :py:meth:`get_disk_utilization_private_img` + ''' + + return self.storage.get_private_img_sz() + + + def get_disk_utilization(self): + '''Return total space actually occuppied by all files belonging to this domain. + + :returns: domain's total disk usage [FIXME unit] + :rtype: FIXME + ''' + + return qubes.utils.get_disk_usage(self.dir_path) + + + def verify_files(self): + '''Verify that files accessed by this machine are sane. + + On success, returns normally. On failure, raises exception. + ''' + + self.storage.verify_files() + + if not os.path.exists(os.path.join(self.kernels_dir, 'vmlinuz')): + raise QubesException( + "VM kernel does not exist: {0}".\ + format(os.path.join(self.kernels_dir, 'vmlinuz'))) + + if not os.path.exists(os.path.join(self.kernels_dir, 'initramfs')): + raise QubesException( + "VM initramfs does not exist: {0}".\ + format(os.path.join(self.kernels_dir, 'initramfs'))) + + self.fire_event('verify-files') + + return True + + + # miscellanous + + def get_start_time(self): + '''Tell when machine was started. + + :rtype: datetime.datetime + ''' + if not self.is_running(): + return None + + # TODO shouldn't this be qubesdb? + start_time = self.app.vmm.xs.read('', "/vm/%s/start_time" % str(self.uuid)) + if start_time != '': + return datetime.datetime.fromtimestamp(float(start_time)) + else: + return None + + + # XXX this probably should go to AppVM + def is_outdated(self): + '''Check whether domain needs restart to update root image from template. + + :returns: :py:obj:`True` if is outdated, :py:obj:`False` otherwise. + :rtype: bool + ''' + + # Makes sense only on VM based on template + if self.template is None: + return False + + if not self.is_running(): + return False + + if not hasattr(self.template, 'rootcow_img'): + return False + + rootimg_inode = os.stat(self.template.root_img) + try: + rootcow_inode = os.stat(self.template.rootcow_img) + except OSError: + # The only case when rootcow_img doesn't exists is in the middle of + # commit_changes, so VM is outdated right now + return True + + current_dmdev = "/dev/mapper/snapshot-{0:x}:{1}-{2:x}:{3}".format( + rootimg_inode[2], rootimg_inode[1], + rootcow_inode[2], rootcow_inode[1]) + + # FIXME + # 51712 (0xCA00) is xvda + # backend node name not available through xenapi :( + used_dmdev = vmm.xs.read('', "/local/domain/0/backend/vbd/{0}/51712/node".format(self.xid)) + + return used_dmdev != current_dmdev + + + def is_networked(self): + '''Check whether this VM can reach network (firewall notwithstanding). + + :returns: :py:obj:`True` if is machine can reach network, :py:obj:`False` otherwise. + :rtype: bool + ''' + + if self.provides_network: + return True + + return self.netvm is not None + + + # + # helper methods + # + + # TODO deprecate `default` + def absolute_path(self, path, default): + '''Return specified path as absolute path. + + Relative paths are relative to :py:attr:`dir_path`. Absolute path are left unchanged. + + :param str path: Path in question (possibly relatve). + :param default: What to return if ``arg`` is :py:obj:`None`. + :returns: Absolute path. + ''' + + if arg is not None and os.path.isabs(arg): + return arg + else: + return os.path.join(self.dir_path, (arg if arg is not None else default)) + + def relative_path(self, path): + '''Return path relative to py:attr:`dir_path`. + + :param str path: Path in question. + :returns: Relative path. + ''' + + return os.path.relpath(path, self.dir_path) +# return arg.replace(self.dir_path + '/', '') + + + def create_qdb_entries(self): + '''Create entries in Qubes DB. + ''' + self.qdb.write("/name", self.name) + self.qdb.write("/qubes-vm-type", self.type) + self.qdb.write("/qubes-vm-updateable", str(self.updateable)) + + if self.provides_network: + self.qdb.write("/qubes-netvm-gateway", self.gateway) + self.qdb.write("/qubes-netvm-secondary-dns", self.secondary_dns) + self.qdb.write("/qubes-netvm-netmask", self.netmask) + self.qdb.write("/qubes-netvm-network", self.network) + + if self.netvm is not None: + self.qdb.write("/qubes-ip", self.ip) + self.qdb.write("/qubes-netmask", self.netvm.netmask) + self.qdb.write("/qubes-gateway", self.netvm.gateway) + self.qdb.write("/qubes-secondary-dns", self.netvm.secondary_dns) + + tzname = qubes.utils.get_timezone() + if tzname: + self.qdb.write("/qubes-timezone", tzname) + + for srv in self.services.keys(): + # convert True/False to "1"/"0" + self.qdb.write("/qubes-service/{0}".format(srv), + str(int(self.services[srv]))) + + self.qdb.write("/qubes-block-devices", '') + + self.qdb.write("/qubes-usb-devices", '') + + self.qdb.write("/qubes-debug-mode", str(int(self.debug))) + + # TODO: Currently the whole qmemman is quite Xen-specific, so stay with + # xenstore for it until decided otherwise + if qmemman_present: + vmm.xs.set_permissions('', '/local/domain/{0}/memory'.format(self.xid), + [{ 'dom': self.xid }]) + + self.fire_event('qdb-created') + + + def _update_libvirt_domain(self): + '''Re-initialise :py:attr:`libvirt_domain`.''' + domain_config = self.create_config_file() + if self._libvirt_domain: + self._libvirt_domain.undefine() + try: + self._libvirt_domain = vmm.libvirt_conn.defineXML(domain_config) + self.uuid = uuid.UUID(bytes=self._libvirt_domain.UUID()) + except libvirt.libvirtError: + if vmm.libvirt_conn.virConnGetLastError()[0] == libvirt.VIR_ERR_NO_DOMAIN: + # accept the fact that libvirt doesn't know anything about this + # domain... + pass + else: + raise + + + def cleanup_vifs(self): + '''Remove stale network device backends. + + Xend does not remove vif when backend domain is down, so we must do it + manually. + ''' + + # FIXME: remove this? + if not self.is_running(): + return + + dev_basepath = '/local/domain/%d/device/vif' % self.xid + for dev in self.app.vmm.xs.ls('', dev_basepath): + # check if backend domain is alive + backend_xid = int(self.app.vmm.xs.read('', '%s/%s/backend-id' % (dev_basepath, dev))) + if backend_xid in self.app.vmm.libvirt_conn.listDomainsID(): + # check if device is still active + if self.app.vmm.xs.read('', '%s/%s/state' % (dev_basepath, dev)) == '4': + continue + # remove dead device + self.app.vmm.xs.rm('', '%s/%s' % (dev_basepath, dev)) + + + + + + + + + + + + # + # workshop -- those are to be reworked later + # + + def get_prefmem(self): + # TODO: qmemman is still xen specific + untrusted_meminfo_key = xs.read('', '/local/domain/%s/memory/meminfo' + % self.xid) + if untrusted_meminfo_key is None or untrusted_meminfo_key == '': + return 0 + domain = qmemman.DomainState(self.xid) + qmemman_algo.refresh_meminfo_for_domain(domain, untrusted_meminfo_key) + domain.memory_maximum = self.get_mem_static_max()*1024 + return qmemman_algo.prefmem(domain)/1024 + + + + # + # landfill -- those are unneeded + # + + + + +# attrs = { + # + ##### Internal attributes - will be overriden in __init__ regardless of args + + # those should be __builtin__.property of something + # used to suppress side effects of clone_attrs + # XXX probably will be obsoleted by .events_enabled +# "_do_not_reset_firewall": { "func": lambda x: False }, + + # XXX WTF? +# "kernels_dir": { +# # for backward compatibility (or another rare case): kernel=None -> kernel in VM dir +# "func": lambda x: \ +# os.path.join(system_path["qubes_kernels_base_dir"], +# self.kernel) if self.kernel is not None \ +# else os.path.join(self.dir_path, +# vm_files["kernels_subdir"]) }, +# "_start_guid_first": { "func": lambda x: False }, +# } + + # this function appears unused +# def _cleanup_zombie_domains(self): +# """ +# This function is workaround broken libxl (which leaves not fully +# created domain on failure) and vchan on domain crash behaviour +# @return: None +# """ +# xc = self.get_xc_dominfo() +# if xc and xc['dying'] == 1: +# # GUID still running? +# guid_pidfile = '/var/run/qubes/guid-running.%d' % xc['domid'] +# if os.path.exists(guid_pidfile): +# guid_pid = open(guid_pidfile).read().strip() +# os.kill(int(guid_pid), 15) +# # qrexec still running? +# if self.is_qrexec_running(): +# #TODO: kill qrexec daemon +# pass diff --git a/qubes/vm/templatevm.py b/qubes/vm/templatevm.py index ec92bdff..0ce694c7 100644 --- a/qubes/vm/templatevm.py +++ b/qubes/vm/templatevm.py @@ -6,8 +6,8 @@ import qubes.vm.qubesvm class TemplateVM(qubes.vm.qubesvm.QubesVM): '''Template for AppVM''' - template = qubes.property('template', - setter=qubes.property.forbidden) - def __init__(self, D): super(TemplateVM, self).__init__(D) + + # Some additional checks for template based VM + assert self.root_img is not None, "Missing root_img for standalone VM!" diff --git a/tests/vm/qubesvm.py b/tests/vm/qubesvm.py new file mode 100644 index 00000000..6273cd46 --- /dev/null +++ b/tests/vm/qubesvm.py @@ -0,0 +1,79 @@ +#!/usr/bin/python2 -O + +import sys +import unittest +sys.path.insert(0, '../../') + +import qubes +import qubes.vm.qubesvm + + +class TestProp(object): + __name__ = 'testprop' + + +class TestVM(object): + def __init__(self): + self.running = False + self.installed_by_rpm = False + + def is_running(self): + return self.running + + +class TC_00_setters(unittest.TestCase): + def setUp(self): + self.vm = TestVM() + self.prop = TestProp() + + + def test_000_setter_qid(self): + self.assertEqual( + qubes.vm.qubesvm._setter_qid(self.vm, self.prop, 5), 5) + + def test_001_setter_qid_lt_0(self): + with self.assertRaises(ValueError): + qubes.vm.qubesvm._setter_qid(self.vm, self.prop, -1) + + def test_002_setter_qid_gt_max(self): + with self.assertRaises(ValueError): + qubes.vm.qubesvm._setter_qid(self.vm, self.prop, qubes.MAX_QID + 5) + + + def test_010_setter_name(self): + self.assertEqual( + qubes.vm.qubesvm._setter_name(self.vm, self.prop, 'test_name-1'), + 'test_name-1') + + def test_011_setter_name_longer_than_31(self): + with self.assertRaises(ValueError): + qubes.vm.qubesvm._setter_name(self.vm, self.prop, 't' * 32) + + def test_012_setter_name_illegal_character(self): + with self.assertRaises(ValueError): + qubes.vm.qubesvm._setter_name(self.vm, self.prop, 'test#') + + def test_013_setter_name_first_not_letter(self): + with self.assertRaises(ValueError): + qubes.vm.qubesvm._setter_name(self.vm, self.prop, '1test') + + def test_014_setter_name_running(self): + self.vm.running = True + with self.assertRaises(qubes.QubesException): + qubes.vm.qubesvm._setter_name(self.vm, self.prop, 'testname') + + def test_015_setter_name_installed_by_rpm(self): + self.vm.installed_by_rpm = True + with self.assertRaises(qubes.QubesException): + qubes.vm.qubesvm._setter_name(self.vm, self.prop, 'testname') + + + @unittest.skip('test not implemented') + def test_020_setter_kernel(self): + pass + + +class TC_90_QubesVM(unittest.TestCase): + @unittest.skip('test not implemented') + def test_000_init(self): + pass From 9fa3d60d0b6b508160f3fd4f5966ad560245f8da Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 29 Dec 2014 13:07:20 +0100 Subject: [PATCH 0027/1004] qubes/events: fix event handling order Events are divided into "pre" and "post" events. "Pre" events fire handlers in MRO, "post" fire them in reverse. --- qubes/__init__.py | 4 ++-- qubes/events.py | 56 ++++++++++++++++++++++++++++++++++---------- qubes/vm/__init__.py | 4 ++-- 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index 6659c324..fe252e6f 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -510,9 +510,9 @@ class property(object): value = self._type(value) if has_oldvalue: - instance.fire_event('property-pre-set:' + self.__name__, value, oldvalue) + instance.fire_event_pre('property-pre-set:' + self.__name__, value, oldvalue) else: - instance.fire_event('property-pre-set:' + self.__name__, value) + instance.fire_event_pre('property-pre-set:' + self.__name__, value) instance._init_property(self, value) diff --git a/qubes/events.py b/qubes/events.py index fa5a9d64..58180f59 100644 --- a/qubes/events.py +++ b/qubes/events.py @@ -92,24 +92,17 @@ class Emitter(object): cls.__handlers__[event].add(handler) - def fire_event(self, event, *args, **kwargs): - '''Call all handlers for an event. + def _fire_event_in_order(self, order, event, *args, **kwargs): + '''Fire event for classes in given order. - Handlers are called for class and all parent classess, in method - resolution order. For each class first are called bound handlers - (specified in class definition), then handlers from extensions. Aside - from above, remaining order is undefined. - - :param str event: event identificator - - All *args* and *kwargs* are passed verbatim. They are different for - different events. + Do not use this method. Use :py:meth:`fire_event` or + :py:meth:`fire_event_pre`. ''' if not self.events_enabled: return - for cls in self.__class__.__mro__: + for cls in order: # first fire bound (= our own) handlers, then handlers from extensions if not hasattr(cls, '__handlers__'): continue @@ -122,3 +115,42 @@ class Emitter(object): # this is from extension or hand-added, so we see method as # unbound, therefore we need to pass self handler(self, event, *args, **kwargs) + + + def fire_event(self, event, *args, **kwargs): + '''Call all handlers for an event. + + Handlers are called for class and all parent classess, in **reversed** + method resolution order. For each class first are called bound handlers + (specified in class definition), then handlers from extensions. Aside + from above, remaining order is undefined. + + .. seealso:: + :py:meth:`fire_event_pre` + + :param str event: event identificator + + All *args* and *kwargs* are passed verbatim. They are different for + different events. + ''' + + self._fire_event_in_order(reversed(self.__class__.__mro__), event, *args, **kwargs) + + + def fire_event_pre(self, event, *args, **kwargs): + '''Call all handlers for an event. + + Handlers are called for class and all parent classess, in **true** + method resolution order. This is intended for ``-pre-`` events, where + order of invocation should be reversed. + + .. seealso:: + :py:meth:`fire_event` + + :param str event: event identificator + + All *args* and *kwargs* are passed verbatim. They are different for + different events. + ''' + + self._fire_event_in_order(self.__class__.__mro__, event, *args, **kwargs) diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index 3a94a0f9..d499d6a6 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -93,7 +93,7 @@ class DeviceCollection(object): raise KeyError( 'device {!r} of class {} already attached to {!r}'.format( device, self._class, self._vm)) - self._vm.fire_event('device-pre-attached:{}'.format(self._class), device) + self._vm.fire_event_pre('device-pre-attached:{}'.format(self._class), device) self._set.add(device) self._vm.fire_event('device-attached:{}'.format(self._class), device) @@ -108,7 +108,7 @@ class DeviceCollection(object): raise KeyError( 'device {!r} of class {} not attached to {!r}'.format( device, self._class, self._vm)) - self._vm.fire_event('device-pre-detached:{}'.format(self._class), device) + self._vm.fire_event_pre('device-pre-detached:{}'.format(self._class), device) self._set.remove(device) self._vm.fire_event('device-detached:{}'.format(self._class), device) From c414ab6df0715c30ebb1c5e4a4a51ac11dfb96f6 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 29 Dec 2014 13:11:05 +0100 Subject: [PATCH 0028/1004] tests: fix VMCollection tests --- tests/init.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/init.py b/tests/init.py index 936d0812..7a428bab 100644 --- a/tests/init.py +++ b/tests/init.py @@ -7,6 +7,7 @@ import lxml.etree sys.path.insert(0, '../') import qubes +import qubes.events import qubes.vm class TC_10_Label(unittest.TestCase): @@ -99,10 +100,13 @@ class TestVM(qubes.vm.BaseVM): name = qubes.property('name') netid = qid +class TestApp(qubes.events.Emitter): + pass + class TC_11_VMCollection(unittest.TestCase): def setUp(self): # XXX passing None may be wrong in the future - self.vms = qubes.VMCollection(None) + self.vms = qubes.VMCollection(TestApp()) self.testvm1 = TestVM(None, None, qid=1, name='testvm1') self.testvm2 = TestVM(None, None, qid=2, name='testvm2') From a82bf7cc548a4d80a2fc70ee689121293a70ec5c Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 5 Jan 2015 14:41:59 +0100 Subject: [PATCH 0029/1004] qubes/tests: Move unit tests inside qubes/, add runner --- qubes/tests/__init__.py | 10 ++++++++++ {tests => qubes/tests}/events.py | 5 +++-- {tests => qubes/tests}/init.py | 11 ++++++----- qubes/tests/run.py | 26 ++++++++++++++++++++++++++ qubes/tests/vm/__init__.py | 0 tests/vm.py => qubes/tests/vm/init.py | 9 +++++---- {tests => qubes/tests}/vm/qubesvm.py | 7 ++++--- 7 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 qubes/tests/__init__.py rename {tests => qubes/tests}/events.py (93%) rename {tests => qubes/tests}/init.py (96%) create mode 100755 qubes/tests/run.py create mode 100644 qubes/tests/vm/__init__.py rename tests/vm.py => qubes/tests/vm/init.py (96%) rename {tests => qubes/tests}/vm/qubesvm.py (94%) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py new file mode 100644 index 00000000..193619df --- /dev/null +++ b/qubes/tests/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/python -O + +import unittest + +class QubesTestCase(unittest.TestCase): + def __str__(self): + return '{}/{}/{}'.format( + '.'.join(self.__class__.__module__.split('.')[2:]), + self.__class__.__name__, + self._testMethodName) diff --git a/tests/events.py b/qubes/tests/events.py similarity index 93% rename from tests/events.py rename to qubes/tests/events.py index 75d94a93..d3bab601 100644 --- a/tests/events.py +++ b/qubes/tests/events.py @@ -3,10 +3,11 @@ import sys import unittest -sys.path.insert(0, '..') import qubes.events -class TC_Emitter(unittest.TestCase): +import qubes.tests + +class TC_00_Emitter(qubes.tests.QubesTestCase): def test_000_add_handler(self): # need something mutable testevent_fired = [False] diff --git a/tests/init.py b/qubes/tests/init.py similarity index 96% rename from tests/init.py rename to qubes/tests/init.py index 7a428bab..58a57729 100644 --- a/tests/init.py +++ b/qubes/tests/init.py @@ -5,12 +5,13 @@ import unittest import lxml.etree -sys.path.insert(0, '../') import qubes import qubes.events import qubes.vm -class TC_10_Label(unittest.TestCase): +import qubes.tests + +class TC_10_Label(qubes.tests.QubesTestCase): def test_000_constructor(self): label = qubes.Label(1, '#cc0000', 'red') @@ -45,7 +46,7 @@ class TestHolder(qubes.PropertyHolder): testprop3 = qubes.property('testprop3', order=2, default='testdefault') testprop4 = qubes.property('testprop4', order=3) -class TC_00_PropertyHolder(unittest.TestCase): +class TC_00_PropertyHolder(qubes.tests.QubesTestCase): def assertXMLEqual(self, xml1, xml2): self.assertEqual(xml1.tag, xml2.tag) self.assertEqual(xml1.text, xml2.text) @@ -103,7 +104,7 @@ class TestVM(qubes.vm.BaseVM): class TestApp(qubes.events.Emitter): pass -class TC_11_VMCollection(unittest.TestCase): +class TC_11_VMCollection(qubes.tests.QubesTestCase): def setUp(self): # XXX passing None may be wrong in the future self.vms = qubes.VMCollection(TestApp()) @@ -203,5 +204,5 @@ class TC_11_VMCollection(unittest.TestCase): # pass -class TC_20_Qubes(unittest.TestCase): +class TC_20_Qubes(qubes.tests.QubesTestCase): pass diff --git a/qubes/tests/run.py b/qubes/tests/run.py new file mode 100755 index 00000000..8a5b703a --- /dev/null +++ b/qubes/tests/run.py @@ -0,0 +1,26 @@ +#!/usr/bin/python -O + +import importlib +import sys +import unittest + +test_order = [ + 'qubes.tests.events', + 'qubes.tests.vm.init', + 'qubes.tests.vm.qubesvm', + 'qubes.tests.init' +] + +sys.path.insert(0, '../../') + +def main(): + suite = unittest.TestSuite() + loader = unittest.TestLoader() + for modname in test_order: + module = importlib.import_module(modname) + suite.addTests(loader.loadTestsFromModule(module)) + + unittest.TextTestRunner(stream=sys.stdout, verbosity=2).run(suite) + +if __name__ == '__main__': + main() diff --git a/qubes/tests/vm/__init__.py b/qubes/tests/vm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/vm.py b/qubes/tests/vm/init.py similarity index 96% rename from tests/vm.py rename to qubes/tests/vm/init.py index f716b652..fc9f87b4 100644 --- a/tests/vm.py +++ b/qubes/tests/vm/init.py @@ -5,11 +5,12 @@ import unittest import lxml.etree -sys.path.insert(0, '../') import qubes import qubes.events import qubes.vm +import qubes.tests + class TestEmitter(qubes.events.Emitter): def __init__(self): @@ -38,7 +39,7 @@ class TestEmitter(qubes.events.Emitter): if self.device_pre_detached_fired: self.device_detached_fired = True -class TC_00_DeviceCollection(unittest.TestCase): +class TC_00_DeviceCollection(qubes.tests.QubesTestCase): def setUp(self): self.emitter = TestEmitter() self.collection = qubes.vm.DeviceCollection(self.emitter, 'testclass') @@ -79,7 +80,7 @@ class TC_00_DeviceCollection(unittest.TestCase): self.collection.detach('testdev') -class TC_01_DeviceManager(unittest.TestCase): +class TC_01_DeviceManager(qubes.tests.QubesTestCase): def setUp(self): self.emitter = TestEmitter() self.manager = qubes.vm.DeviceManager(self.emitter) @@ -99,7 +100,7 @@ class TestVM(qubes.vm.BaseVM): testlabel = qubes.property('testlabel') defaultprop = qubes.property('defaultprop', default='defaultvalue') -class TC_10_BaseVM(unittest.TestCase): +class TC_10_BaseVM(qubes.tests.QubesTestCase): def setUp(self): self.xml = lxml.etree.XML(''' diff --git a/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py similarity index 94% rename from tests/vm/qubesvm.py rename to qubes/tests/vm/qubesvm.py index 6273cd46..02750824 100644 --- a/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -2,11 +2,12 @@ import sys import unittest -sys.path.insert(0, '../../') import qubes import qubes.vm.qubesvm +import qubes.tests + class TestProp(object): __name__ = 'testprop' @@ -21,7 +22,7 @@ class TestVM(object): return self.running -class TC_00_setters(unittest.TestCase): +class TC_00_setters(qubes.tests.QubesTestCase): def setUp(self): self.vm = TestVM() self.prop = TestProp() @@ -73,7 +74,7 @@ class TC_00_setters(unittest.TestCase): pass -class TC_90_QubesVM(unittest.TestCase): +class TC_90_QubesVM(qubes.tests.QubesTestCase): @unittest.skip('test not implemented') def test_000_init(self): pass From 613b03d2778c9e0aef115f3e543c4a07aa8fa423 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 5 Jan 2015 15:39:14 +0100 Subject: [PATCH 0030/1004] qubes/tests: Common TestEmitter class qubes.tests.TestEmitter is intended to check whether specific event fired on given emitter. --- qubes/tests/__init__.py | 82 +++++++++++++++++++++++++++++++++++++++++ qubes/tests/vm/init.py | 49 ++++++------------------ 2 files changed, 93 insertions(+), 38 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 193619df..9fec048a 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1,10 +1,92 @@ #!/usr/bin/python -O +import collections import unittest +import qubes.events + +class TestEmitter(qubes.events.Emitter): + '''Dummy event emitter which records events fired on it. + + Events are counted in :py:attr:`fired_events` attribute, which is + :py:class:`collections.Counter` instance. For each event, ``(event, args, + kwargs)`` object is counted. *event* is event name (a string), *args* is + tuple with positional arguments and *kwargs* is sorted tuple of items from + keyword arguments. + + >>> emitter = TestEmitter() + >>> emitter.fired_events + Counter() + >>> emitter.fire_event('event', 1, 2, 3, spam='eggs', foo='bar') + >>> emitter.fired_events + Counter({('event', (1, 2, 3), (('foo', 'bar'), ('spam', 'eggs'))): 1}) + ''' + def __init__(self, *args, **kwargs): + super(TestEmitter, self).__init__(*args, **kwargs) + + #: :py:class:`collections.Counter` instance + self.fired_events = collections.Counter() + + def fire_event(self, event, *args, **kwargs): + super(TestEmitter, self).fire_event(event, *args, **kwargs) + self.fired_events[(event, args, tuple(sorted(kwargs.items())))] += 1 + + def fire_event_pre(self, event, *args, **kwargs): + super(TestEmitter, self).fire_event_pre(event, *args, **kwargs) + self.fired_events[(event, args, tuple(sorted(kwargs.items())))] += 1 + + class QubesTestCase(unittest.TestCase): + '''Base class for Qubes unit tests. + + ''' def __str__(self): return '{}/{}/{}'.format( '.'.join(self.__class__.__module__.split('.')[2:]), self.__class__.__name__, self._testMethodName) + + + def assertEventFired(self, emitter, event, args=[], kwargs=[]): + '''Check whether event was fired on given emitter and fail if it did + not. + + :param TestEmitter emitter: emitter which is being checked + :param str event: event identifier + :param list args: when given, all items must appear in args passed to event + :param list kwargs: when given, all items must appear in kwargs passed to event + ''' + + for ev, ev_args, ev_kwargs in emitter.fired_events: + if ev != event: + continue + if any(i not in ev_args for i in args): + continue + if any(i not in ev_kwargs for i in kwargs): + continue + + return + + self.fail('event {!r} did not fire on {!r}'.format(event, emitter)) + + + def assertEventNotFired(self, emitter, event, args=[], kwargs=[]): + '''Check whether event was fired on given emitter. Fail if it did. + + :param TestEmitter emitter: emitter which is being checked + :param str event: event identifier + :param list args: when given, all items must appear in args passed to event + :param list kwargs: when given, all items must appear in kwargs passed to event + ''' + + for ev, ev_args, ev_kwargs in emitter.fired_events: + if ev != event: + continue + if any(i not in ev_args for i in args): + continue + if any(i not in ev_kwargs for i in kwargs): + continue + + self.fail('event {!r} did fire on {!r}'.format(event, emitter)) + + return diff --git a/qubes/tests/vm/init.py b/qubes/tests/vm/init.py index fc9f87b4..1608f189 100644 --- a/qubes/tests/vm/init.py +++ b/qubes/tests/vm/init.py @@ -12,36 +12,9 @@ import qubes.vm import qubes.tests -class TestEmitter(qubes.events.Emitter): - def __init__(self): - super(TestEmitter, self).__init__() - self.device_pre_attached_fired = False - self.device_attached_fired = False - self.device_pre_detached_fired = False - self.device_detached_fired = False - - @qubes.events.handler('device-pre-attached:testclass') - def on_device_pre_attached(self, event, dev): - self.device_pre_attached_fired = True - - @qubes.events.handler('device-attached:testclass') - def on_device_attached(self, event, dev): - if self.device_pre_attached_fired: - self.device_attached_fired = True - - @qubes.events.handler('device-pre-detached:testclass') - def on_device_pre_detached(self, event, dev): - if self.device_attached_fired: - self.device_pre_detached_fired = True - - @qubes.events.handler('device-detached:testclass') - def on_device_detached(self, event, dev): - if self.device_pre_detached_fired: - self.device_detached_fired = True - class TC_00_DeviceCollection(qubes.tests.QubesTestCase): def setUp(self): - self.emitter = TestEmitter() + self.emitter = qubes.tests.TestEmitter() self.collection = qubes.vm.DeviceCollection(self.emitter, 'testclass') def test_000_init(self): @@ -49,18 +22,18 @@ class TC_00_DeviceCollection(qubes.tests.QubesTestCase): def test_001_attach(self): self.collection.attach('testdev') - self.assertTrue(self.emitter.device_pre_attached_fired) - self.assertTrue(self.emitter.device_attached_fired) - self.assertFalse(self.emitter.device_pre_detached_fired) - self.assertFalse(self.emitter.device_detached_fired) + self.assertEventFired(self.emitter, 'device-pre-attached:testclass') + self.assertEventFired(self.emitter, 'device-attached:testclass') + self.assertEventNotFired(self.emitter, 'device-pre-detached:testclass') + self.assertEventNotFired(self.emitter, 'device-detached:testclass') def test_002_detach(self): self.collection.attach('testdev') self.collection.detach('testdev') - self.assertTrue(self.emitter.device_pre_attached_fired) - self.assertTrue(self.emitter.device_attached_fired) - self.assertTrue(self.emitter.device_pre_detached_fired) - self.assertTrue(self.emitter.device_detached_fired) + self.assertEventFired(self.emitter, 'device-pre-attached:testclass') + self.assertEventFired(self.emitter, 'device-attached:testclass') + self.assertEventFired(self.emitter, 'device-pre-detached:testclass') + self.assertEventFired(self.emitter, 'device-detached:testclass') def test_010_empty_detach(self): with self.assertRaises(LookupError): @@ -82,7 +55,7 @@ class TC_00_DeviceCollection(qubes.tests.QubesTestCase): class TC_01_DeviceManager(qubes.tests.QubesTestCase): def setUp(self): - self.emitter = TestEmitter() + self.emitter = qubes.tests.TestEmitter() self.manager = qubes.vm.DeviceManager(self.emitter) def test_000_init(self): @@ -90,7 +63,7 @@ class TC_01_DeviceManager(qubes.tests.QubesTestCase): def test_001_missing(self): self.manager['testclass'].attach('testdev') - self.assertTrue(self.emitter.device_attached_fired) + self.assertEventFired(self.emitter, 'device-attached:testclass') class TestVM(qubes.vm.BaseVM): From 8ace0fa63474a930ee7b436e9a85b9e3e04be512 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 5 Jan 2015 17:01:13 +0100 Subject: [PATCH 0031/1004] qubes/tests: skipping tests outside dom0 New variable and decorator enable skipping tests outside dom0, that is, without connection to libvirtd. --- qubes/tests/__init__.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 9fec048a..c702e0b4 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -3,8 +3,31 @@ import collections import unittest +import qubes.config import qubes.events + +#: :py:obj:`True` if running in dom0, :py:obj:`False` otherwise +in_dom0 = False + +try: + import libvirt + libvirt.openReadOnly(qubes.config.defaults['libvirt_uri']).close() + in_dom0 = True + del libvirt +except libvirt.libvirtError: + pass + + +def skipUnlessDom0(test_item): + '''Decorator that skips test outside dom0. + + Some tests (especially integration tests) have to be run in more or less + working dom0. This is checked by connecting to libvirt. + ''' + return unittest.skipUnless(in_dom0, 'outside dom0')(test_item) + + class TestEmitter(qubes.events.Emitter): '''Dummy event emitter which records events fired on it. From de9eb60f61e2cc177bf4001c4b09440b55923607 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 5 Jan 2015 19:15:32 +0100 Subject: [PATCH 0032/1004] Developer's documentation for qubes.tests --- doc/index.rst | 1 + doc/qubes-tests.rst | 128 ++++++++++++++++++++++++++++++++++++++++ qubes/tests/__init__.py | 10 +++- 3 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 doc/qubes-tests.rst diff --git a/doc/index.rst b/doc/index.rst index 525a3d00..9552e67e 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -32,6 +32,7 @@ Developer documentation qubes-plugins qubes-ext qubes-log + qubes-tests qubes-dochelpers Indices and tables diff --git a/doc/qubes-tests.rst b/doc/qubes-tests.rst new file mode 100644 index 00000000..fda28a37 --- /dev/null +++ b/doc/qubes-tests.rst @@ -0,0 +1,128 @@ +:py:mod:`qubes.tests` -- Writing tests for qubes +================================================ + +Writing tests is very important for ensuring quality of code that is delivered. +Given test case may check for variety of conditions, but they generally fall +inside those two categories of conformance tests: + +* Unit tests: these test smallest units of code, probably methods of functions, + or even combination of arguments for one specific method. + +* Integration tests: these test interworking of units. + +We are interested in both categories. + +There is also distinguished category of regression tests (both unit- and +integration-level), which are included because they check for specific bugs that +were fixed in the past and should not happen in the future. Those should be +accompanied with reference to closed ticked that describes the bug. + +Qubes' tests are written using :py:mod:`unittest` module from Python Standard +Library for both unit test and integration tests. + +Test case organisation +---------------------- + +Every module (like :py:mod:`qubes.vm.qubesvm`) should have its companion (like +``qubes.tests.vm.qubesvm``). Packages ``__init__.py`` files should be +accompanied by ``init.py`` inside respective directory under :file:`tests/`. +Inside tests module there should be one :py:class:`qubes.tests.QubesTestCase` +class for each class in main module plus one class for functions and global +variables. :py:class:`qubes.tests.QubesTestCase` classes should be named +``TC_xx_ClassName``, where ``xx`` is two-digit number. Test functions should be +named ``test_xxx_test_name``, where ``xxx`` is three-digit number. You may +introduce some structure of your choice in this number. + +FIXME: where are placed integration tests? + +Writing tests +------------- + +First of all, testing is art, not science. Testing is not panaceum and won't +solve all of your problems. Rules given in this guide and elsewhere should be +followed, but shouldn't be worshipped. + +When writing test, you should think about order of execution. Tests should be +written bottom-to-top, that is, tests that are ran later may depend on features +that are tested after but not the other way around. This is important, because +when encountering failure we expect the reason happen *before*, and not after +failure occured. Therefore, when encountering multiple errors, we may instantly +focus on fixing the first one and not wondering if any later problems may be +relevant or not. This is the reason of numbers in names of the classes and test +methods. + +You may, when it makes sense, manipulate private members of classes under tests. +This violates one of the founding principles of object-oriented programming, but +may be required to write tests in correct order if your class provides public +methods with circular dependencies. For example containers may check if added +item is already in container, but you can't test ``__contains__`` method without +something already inside. Don't forget to test the other method later. + +Special Qubes-specific considerations +------------------------------------- + +Events +^^^^^^ + +:py:class:`qubes.tests.QubesTestCase` provides convenient methods for checking +if event fired or not: :py:meth:`qubes.tests.QubesTestCase.assertEventFired` and +:py:meth:`qubes.tests.QubesTestCase.assertEventNotFired`. These require that +emitter is subclass of :py:class:`qubes.tests.TestEmitter`. You may instantiate +it directly:: + + import qubes.tests + + class TC_10_SomeClass(qubes.tests.QubesTestCase): + def test_000_event(self): + emitter = qubes.tests.TestEmitter() + emitter.fire_event('did-fire') + self.assertEventFired(emitter, 'did-fire') + +If you need to snoop specific class (which already is a child of +:py:class:`qubes.events.Emitter`, possibly indirect), you can define derivative +class which uses :py:class:`qubes.tests.TestEmitter` as mix-in:: + + import qubes + import qubes.tests + + class TestHolder(qubes.tests.TestEmitter, qubes.PropertyHolder): + pass + + class TC_20_PropertyHolder(qubes.tests.QubesTestCase): + def test_000_event(self): + emitter = TestHolder() + self.assertEventNotFired(emitter, 'did-not-fire') + +Dom0 +^^^^ + +Qubes is a complex piece of software and depends on number other complex pieces, +notably VM hypervisor or some other isolation provider. Not everything may be +testable under all conditions. Some tests (mainly unit tests) are expected to +run during compilation, but many tests (probably all of the integration tests +and more) can run only inside already deployed Qubes installation. There is +special decorator, :py:func:`qubes.tests.skipUnlessDom0` which causes test (or +even entire class) to be skipped outside dom0. Use it freely:: + + import qubes.tests + + class TC_30_SomeClass(qubes.tests.QubesTestCase): + @qubes.tests.skipUnlessDom0 + def test_000_inside_dom0(self): + # this is skipped outside dom0 + pass + + @qubes.tests.skipUnlessDom0 + class TC_31_SomeOtherClass(qubes.tests.QubesTestCase): + # all tests in this class are skipped + pass + + +Module contents +--------------- + +.. automodule:: qubes.tests + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et tw=80 diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index c702e0b4..4ac813b9 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -25,6 +25,7 @@ def skipUnlessDom0(test_item): Some tests (especially integration tests) have to be run in more or less working dom0. This is checked by connecting to libvirt. ''' + return unittest.skipUnless(in_dom0, 'outside dom0')(test_item) @@ -44,6 +45,7 @@ class TestEmitter(qubes.events.Emitter): >>> emitter.fired_events Counter({('event', (1, 2, 3), (('foo', 'bar'), ('spam', 'eggs'))): 1}) ''' + def __init__(self, *args, **kwargs): super(TestEmitter, self).__init__(*args, **kwargs) @@ -61,8 +63,8 @@ class TestEmitter(qubes.events.Emitter): class QubesTestCase(unittest.TestCase): '''Base class for Qubes unit tests. - ''' + def __str__(self): return '{}/{}/{}'.format( '.'.join(self.__class__.__module__.split('.')[2:]), @@ -74,7 +76,8 @@ class QubesTestCase(unittest.TestCase): '''Check whether event was fired on given emitter and fail if it did not. - :param TestEmitter emitter: emitter which is being checked + :param emitter: emitter which is being checked + :type emitter: :py:class:`TestEmitter` :param str event: event identifier :param list args: when given, all items must appear in args passed to event :param list kwargs: when given, all items must appear in kwargs passed to event @@ -96,7 +99,8 @@ class QubesTestCase(unittest.TestCase): def assertEventNotFired(self, emitter, event, args=[], kwargs=[]): '''Check whether event was fired on given emitter. Fail if it did. - :param TestEmitter emitter: emitter which is being checked + :param emitter: emitter which is being checked + :type emitter: :py:class:`TestEmitter` :param str event: event identifier :param list args: when given, all items must appear in args passed to event :param list kwargs: when given, all items must appear in kwargs passed to event From 52c1be49ecc31d60334211062da3900423bcf602 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Wed, 7 Jan 2015 15:35:39 +0100 Subject: [PATCH 0033/1004] qubes/vm: remove unneccessary import --- qubes/vm/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index d499d6a6..e878642e 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -53,7 +53,6 @@ import collections import functools import sys -import dateutil.parser import lxml.etree import qubes From eabc5711021cb01655b26007522bfccb8fefa806 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Wed, 7 Jan 2015 16:46:59 +0100 Subject: [PATCH 0034/1004] qubes/tests: colourful test runner --- qubes/tests/run.py | 179 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 177 insertions(+), 2 deletions(-) diff --git a/qubes/tests/run.py b/qubes/tests/run.py index 8a5b703a..0d3d9080 100755 --- a/qubes/tests/run.py +++ b/qubes/tests/run.py @@ -1,5 +1,6 @@ #!/usr/bin/python -O +import curses import importlib import sys import unittest @@ -13,6 +14,178 @@ test_order = [ sys.path.insert(0, '../../') +class ANSIColor(dict): + def __init__(self): + super(ANSIColor, self).__init__() + try: + curses.setupterm() + except curses.error: + return + + self['black'] = curses.tparm(curses.tigetstr('setaf'), 0) + self['red'] = curses.tparm(curses.tigetstr('setaf'), 1) + self['green'] = curses.tparm(curses.tigetstr('setaf'), 2) + self['yellow'] = curses.tparm(curses.tigetstr('setaf'), 3) + self['blue'] = curses.tparm(curses.tigetstr('setaf'), 4) + self['magenta'] = curses.tparm(curses.tigetstr('setaf'), 5) + self['cyan'] = curses.tparm(curses.tigetstr('setaf'), 6) + self['white'] = curses.tparm(curses.tigetstr('setaf'), 7) + + self['bold'] = curses.tigetstr('bold') + self['normal'] = curses.tigetstr('sgr0') + + def __missing__(self, key): + return '' + + +class ANSITestResult(unittest.TestResult): + '''A test result class that can print colourful text results to a stream. + + Used by TextTestRunner. This is a lightly rewritten unittest.TextTestResult. + ''' + + separator1 = unittest.TextTestResult.separator1 + separator2 = unittest.TextTestResult.separator2 + + def __init__(self, stream, descriptions, verbosity): + super(ANSITestResult, self).__init__(stream, descriptions, verbosity) + self.stream = stream + self.showAll = verbosity > 1 + self.dots = verbosity == 1 + self.descriptions = descriptions + + self.color = ANSIColor() + + def _fmtexc(self, err): + s = str(err[1]) + if s: + return '{bold}{}:{normal} {!s}'.format( + err[0].__name__, err[1], **self.color) + else: + return '{bold}{}{normal}'.format(err[0].__name__, **self.color) + + def getDescription(self, test): + teststr = str(test).split('/') + teststr[-1] = '{bold}{}{normal}'.format(teststr[-1], **self.color) + teststr = '/'.join(teststr) + + doc_first_line = test.shortDescription() + if self.descriptions and doc_first_line: + return '\n'.join((teststr, ' {}'.format( + doc_first_line, **self.color))) + else: + return teststr + + def startTest(self, test): + super(ANSITestResult, self).startTest(test) + if self.showAll: + self.stream.write(self.getDescription(test)) + self.stream.write(' ... ') + self.stream.flush() + + def addSuccess(self, test): + super(ANSITestResult, self).addSuccess(test) + if self.showAll: + self.stream.writeln('{green}ok{normal}'.format(**self.color)) + elif self.dots: + self.stream.write('.') + self.stream.flush() + + def addError(self, test, err): + super(ANSITestResult, self).addError(test, err) + if self.showAll: + self.stream.writeln('{red}{bold}ERROR{normal} ({})'.format( + self._fmtexc(err), **self.color)) + elif self.dots: + self.stream.write('{red}{bold}E{normal}'.format(**self.color)) + self.stream.flush() + + def addFailure(self, test, err): + super(ANSITestResult, self).addFailure(test, err) + if self.showAll: + self.stream.writeln('{red}FAIL{normal}'.format(**self.color)) + elif self.dots: + self.stream.write('{red}F{normal}'.format(**self.color)) + self.stream.flush() + + def addSkip(self, test, reason): + super(ANSITestResult, self).addSkip(test, reason) + if self.showAll: + self.stream.writeln('{cyan}skipped{normal} ({})'.format( + reason, **self.color)) + elif self.dots: + self.stream.write('{cyan}s{normal}'.format(**self.color)) + self.stream.flush() + + def addExpectedFailure(self, test, err): + super(ANSITestResult, self).addExpectedFailure(test, err) + if self.showAll: + self.stream.writeln('{yellow}expected failure{normal}'.format( + **self.color)) + elif self.dots: + self.stream.write('{yellow}x{normal}'.format(**self.color)) + self.stream.flush() + + def addUnexpectedSuccess(self, test): + super(ANSITestResult, self).addUnexpectedSuccess(test) + if self.showAll: + self.stream.writeln( + '{yellow}{bold}unexpected success{normal}'.format(**self.color)) + elif self.dots: + self.stream.write('{yellow}{bold}u{normal}'.format(**self.color)) + self.stream.flush() + + def printErrors(self): + if self.dots or self.showAll: + self.stream.writeln() + self.printErrorList( + '{red}{bold}ERROR{normal}'.format(**self.color), self.errors) + self.printErrorList( + '{red}FAIL{normal}'.format(**self.color), self.failures) + + def printErrorList(self, flavour, errors): + for test, err in errors: + self.stream.writeln(self.separator1) + self.stream.writeln('%s: %s' % (flavour,self.getDescription(test))) + self.stream.writeln(self.separator2) + self.stream.writeln('%s' % err) + + +def demo(verbosity=2): + import qubes.tests + class TC_Demo(qubes.tests.QubesTestCase): + '''Demo class''' + def test_0_success(self): + '''Demo test (success)''' + pass + def test_1_error(self): + '''Demo test (error)''' + raise Exception() + def test_2_failure(self): + '''Demo test (failure)''' + self.fail('boo') + def test_3_skip(self): + '''Demo test (skipped by call to self.skipTest())''' + self.skipTest('skip') + @unittest.skip(None) + def test_4_skip_decorator(self): + '''Demo test (skipped by decorator)''' + pass + @unittest.expectedFailure + def test_5_expected_failure(self): + '''Demo test (expected failure)''' + self.fail() + @unittest.expectedFailure + def test_6_unexpected_success(self): + '''Demo test (unexpected success)''' + pass + + suite = unittest.TestLoader().loadTestsFromTestCase(TC_Demo) + runner = unittest.TextTestRunner(stream=sys.stdout, verbosity=verbosity) + runner.resultclass = ANSITestResult + return runner.run(suite).wasSuccessful() + + def main(): suite = unittest.TestSuite() loader = unittest.TestLoader() @@ -20,7 +193,9 @@ def main(): module = importlib.import_module(modname) suite.addTests(loader.loadTestsFromModule(module)) - unittest.TextTestRunner(stream=sys.stdout, verbosity=2).run(suite) + runner = unittest.TextTestRunner(stream=sys.stdout, verbosity=2) + runner.resultclass = ANSITestResult + return runner.run(suite).wasSuccessful() if __name__ == '__main__': - main() + sys.exit(not main()) From b442fb0fab882ce5efdb799308cc373f6a51820b Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Thu, 8 Jan 2015 16:48:19 +0100 Subject: [PATCH 0035/1004] qubes/doc: Test's documentation improvement --- doc/qubes-tests.rst | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/doc/qubes-tests.rst b/doc/qubes-tests.rst index fda28a37..a5dd06ec 100644 --- a/doc/qubes-tests.rst +++ b/doc/qubes-tests.rst @@ -42,14 +42,30 @@ First of all, testing is art, not science. Testing is not panaceum and won't solve all of your problems. Rules given in this guide and elsewhere should be followed, but shouldn't be worshipped. -When writing test, you should think about order of execution. Tests should be -written bottom-to-top, that is, tests that are ran later may depend on features +Test can be divided into three phases. The first part is setup phase. In this +part you should arrange for a test condition to occur. You intentionally put +system under test in some specific state. Phase two is executing test condition +-- for example you check some variable for equality or expect that some +exception is raised. Phase three is responsible for returning a verdict. This is +largely done by the framework. + +When writing test, you should think about order of execution. This is the reason +of numbers in names of the classes and test methods. Tests should be written +bottom-to-top, that is, test setups that are ran later may depend on features that are tested after but not the other way around. This is important, because when encountering failure we expect the reason happen *before*, and not after failure occured. Therefore, when encountering multiple errors, we may instantly focus on fixing the first one and not wondering if any later problems may be -relevant or not. This is the reason of numbers in names of the classes and test -methods. +relevant or not. Some people also like to enable +:py:attr:`unittest.TestResult.failfast` feature, which stops on the first failed +test -- with wrong order this messes up their workflow. + +Test should fail for one reason only and test one specific issue. This does not +mean that you can use one ``.assert*`` method per ``test_`` function: for +example when testing one regular expression you are welcome to test many valid +and/or invalid inputs, especcialy when test setup is complicated. However, if +you encounter problems during setup phase, you should *skip* the test, and not +fail it. This also aids interpretation of results. You may, when it makes sense, manipulate private members of classes under tests. This violates one of the founding principles of object-oriented programming, but @@ -86,7 +102,7 @@ class which uses :py:class:`qubes.tests.TestEmitter` as mix-in:: import qubes.tests class TestHolder(qubes.tests.TestEmitter, qubes.PropertyHolder): - pass + pass class TC_20_PropertyHolder(qubes.tests.QubesTestCase): def test_000_event(self): From 74c3126b809c1f396c73f2ade968f83e329d8279 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Thu, 8 Jan 2015 17:29:41 +0100 Subject: [PATCH 0036/1004] qubes/tests: add some tests for qubes.property --- qubes/tests/init.py | 108 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 4 deletions(-) diff --git a/qubes/tests/init.py b/qubes/tests/init.py index 58a57729..56ff2923 100644 --- a/qubes/tests/init.py +++ b/qubes/tests/init.py @@ -11,7 +11,7 @@ import qubes.vm import qubes.tests -class TC_10_Label(qubes.tests.QubesTestCase): +class TC_00_Label(qubes.tests.QubesTestCase): def test_000_constructor(self): label = qubes.Label(1, '#cc0000', 'red') @@ -40,13 +40,113 @@ class TC_10_Label(qubes.tests.QubesTestCase): self.assertEqual(label.icon_dispvm, 'dispvm-red') +class TC_10_property(qubes.tests.QubesTestCase): + def setUp(self): + try: + class TestHolder(qubes.tests.TestEmitter, qubes.PropertyHolder): + testprop1 = qubes.property('testprop1') + except: + self.skipTest('TestHolder class definition failed') + self.holder = TestHolder(None) + + def test_000_init(self): + pass + + def test_001_hash(self): + hash(self.holder.__class__.testprop1) + + def test_002_eq(self): + self.assertEquals(qubes.property('testprop2'), + qubes.property('testprop2')) + + def test_010_set(self): + self.holder.testprop1 = 'testvalue' + self.assertEventFired(self.holder, 'property-pre-set:testprop1') + self.assertEventFired(self.holder, 'property-set:testprop1') + + def test_020_get(self): + self.holder.testprop1 = 'testvalue' + self.assertEqual(self.holder.testprop1, 'testvalue') + + def test_021_get_unset(self): + with self.assertRaises(AttributeError): + self.holder.testprop1 + + def test_022_get_default(self): + class TestHolder(qubes.tests.TestEmitter, qubes.PropertyHolder): + testprop1 = qubes.property('testprop1', default='defaultvalue') + holder = TestHolder(None) + + self.assertEqual(holder.testprop1, 'defaultvalue') + + def test_023_get_default_func(self): + class TestHolder(qubes.tests.TestEmitter, qubes.PropertyHolder): + testprop1 = qubes.property('testprop1', default=(lambda self: 'defaultvalue')) + holder = TestHolder(None) + + self.assertEqual(holder.testprop1, 'defaultvalue') + holder.testprop1 = 'testvalue' + self.assertEqual(holder.testprop1, 'testvalue') + + def test_030_set_setter(self): + def setter(self2, prop, value): + self.assertIs(self2, holder) + self.assertIs(prop, TestHolder.testprop1) + self.assertEquals(value, 'testvalue') + return 'settervalue' + class TestHolder(qubes.tests.TestEmitter, qubes.PropertyHolder): + testprop1 = qubes.property('testprop1', setter=setter) + holder = TestHolder(None) + + holder.testprop1 = 'testvalue' + self.assertEqual(holder.testprop1, 'settervalue') + + def test_031_set_type(self): + class TestHolder(qubes.tests.TestEmitter, qubes.PropertyHolder): + testprop1 = qubes.property('testprop1', type=int) + holder = TestHolder(None) + + holder.testprop1 = '5' + self.assertEqual(holder.testprop1, 5) + self.assertNotEqual(holder.testprop1, '5') + + def test_090_delete(self): + self.holder.testprop1 = 'testvalue' + try: + if self.holder.testprop1 != 'testvalue': + self.skipTest('testprop1 value is wrong') + except AttributeError: + self.skipTest('testprop1 value is wrong') + + del self.holder.testprop1 + + with self.assertRaises(AttributeError): + self.holder.testprop + + def test_091_delete_default(self): + class TestHolder(qubes.tests.TestEmitter, qubes.PropertyHolder): + testprop1 = qubes.property('testprop1', default='defaultvalue') + holder = TestHolder(None) + holder.testprop1 = 'testvalue' + + try: + if holder.testprop1 != 'testvalue': + self.skipTest('testprop1 value is wrong') + except AttributeError: + self.skipTest('testprop1 value is wrong') + + del holder.testprop1 + + self.assertEqual(holder.testprop1, 'defaultvalue') + + class TestHolder(qubes.PropertyHolder): testprop1 = qubes.property('testprop1', order=0) testprop2 = qubes.property('testprop2', order=1, save_via_ref=True) testprop3 = qubes.property('testprop3', order=2, default='testdefault') testprop4 = qubes.property('testprop4', order=3) -class TC_00_PropertyHolder(qubes.tests.QubesTestCase): +class TC_20_PropertyHolder(qubes.tests.QubesTestCase): def assertXMLEqual(self, xml1, xml2): self.assertEqual(xml1.tag, xml2.tag) self.assertEqual(xml1.text, xml2.text) @@ -104,7 +204,7 @@ class TestVM(qubes.vm.BaseVM): class TestApp(qubes.events.Emitter): pass -class TC_11_VMCollection(qubes.tests.QubesTestCase): +class TC_30_VMCollection(qubes.tests.QubesTestCase): def setUp(self): # XXX passing None may be wrong in the future self.vms = qubes.VMCollection(TestApp()) @@ -204,5 +304,5 @@ class TC_11_VMCollection(qubes.tests.QubesTestCase): # pass -class TC_20_Qubes(qubes.tests.QubesTestCase): +class TC_90_Qubes(qubes.tests.QubesTestCase): pass From 713ced8cd27b0ca88f2318c2e461f58db638a579 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Thu, 8 Jan 2015 17:32:45 +0100 Subject: [PATCH 0037/1004] qubes/tests: add event asserts for qubes module --- qubes/tests/__init__.py | 15 +++++++++++++++ qubes/tests/init.py | 24 +++++++++++++----------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 4ac813b9..00721fcc 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -72,6 +72,21 @@ class QubesTestCase(unittest.TestCase): self._testMethodName) + def assertXMLEqual(self, xml1, xml2): + '''Check for equality of two XML objects. + + :param xml1: first element + :param xml2: second element + :type xml1: :py:class:`lxml.etree._Element` + :type xml2: :py:class:`lxml.etree._Element` + ''' + self.assertEqual(xml1.tag, xml2.tag) + self.assertEqual(xml1.text, xml2.text) + self.assertItemsEqual(xml1.keys(), xml2.keys()) + for key in xml1.keys(): + self.assertEqual(xml1.get(key), xml2.get(key)) + + def assertEventFired(self, emitter, event, args=[], kwargs=[]): '''Check whether event was fired on given emitter and fail if it did not. diff --git a/qubes/tests/init.py b/qubes/tests/init.py index 56ff2923..78473bfa 100644 --- a/qubes/tests/init.py +++ b/qubes/tests/init.py @@ -140,20 +140,14 @@ class TC_10_property(qubes.tests.QubesTestCase): self.assertEqual(holder.testprop1, 'defaultvalue') -class TestHolder(qubes.PropertyHolder): +class TestHolder(qubes.tests.TestEmitter, qubes.PropertyHolder): testprop1 = qubes.property('testprop1', order=0) testprop2 = qubes.property('testprop2', order=1, save_via_ref=True) testprop3 = qubes.property('testprop3', order=2, default='testdefault') testprop4 = qubes.property('testprop4', order=3) -class TC_20_PropertyHolder(qubes.tests.QubesTestCase): - def assertXMLEqual(self, xml1, xml2): - self.assertEqual(xml1.tag, xml2.tag) - self.assertEqual(xml1.text, xml2.text) - self.assertEqual(sorted(xml1.keys()), sorted(xml2.keys())) - for key in xml1.keys(): - self.assertEqual(xml1.get(key), xml2.get(key)) +class TC_20_PropertyHolder(qubes.tests.QubesTestCase): def setUp(self): xml = lxml.etree.XML(''' @@ -166,8 +160,13 @@ class TC_20_PropertyHolder(qubes.tests.QubesTestCase): self.holder = TestHolder(xml) + def test_000_load_properties(self): self.holder.load_properties() + + self.assertEventFired(self.holder, 'property-loaded') + self.assertEventNotFired(self.holder, 'property-set:testprop1') + self.assertEquals(self.holder.testprop1, 'testvalue1') self.assertEquals(self.holder.testprop2, 'testref2') self.assertEquals(self.holder.testprop3, 'testdefault') @@ -201,13 +200,13 @@ class TestVM(qubes.vm.BaseVM): name = qubes.property('name') netid = qid -class TestApp(qubes.events.Emitter): +class TestApp(qubes.tests.TestEmitter): pass class TC_30_VMCollection(qubes.tests.QubesTestCase): def setUp(self): - # XXX passing None may be wrong in the future - self.vms = qubes.VMCollection(TestApp()) + self.app = TestApp() + self.vms = qubes.VMCollection(self.app) self.testvm1 = TestVM(None, None, qid=1, name='testvm1') self.testvm2 = TestVM(None, None, qid=2, name='testvm2') @@ -234,6 +233,8 @@ class TC_30_VMCollection(qubes.tests.QubesTestCase): self.vms.add(self.testvm1) self.assertIn(1, self.vms) + self.assertEventFired(self.app, 'domain-added', args=[self.testvm1]) + with self.assertRaises(TypeError): self.vms.add(object()) @@ -284,6 +285,7 @@ class TC_30_VMCollection(qubes.tests.QubesTestCase): del self.vms['testvm2'] self.assertItemsEqual(self.vms.vms(), [self.testvm1]) + self.assertEventFired(self.app, 'domain-deleted', args=[self.testvm2]) def test_100_get_new_unused_qid(self): self.vms.add(self.testvm1) From 6ec86ec9f7c8240cedf6e05795140e5e671fc9e8 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Thu, 8 Jan 2015 17:42:34 +0100 Subject: [PATCH 0038/1004] qubes: property may be unset by assinging DEFAULT Introducing qubes.property.DEFAULT special value, which may be assigned to any property. Result is the same as del'ing a property. --- qubes/__init__.py | 13 +++++++++++++ qubes/tests/init.py | 15 ++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index fe252e6f..42553c7f 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -430,6 +430,11 @@ class property(object): This class holds one property that can be saved to and loaded from :file:`qubes.xml`. It is used for both global and per-VM properties. + Property can be unset by ordinary ``del`` statement or assigning + :py:attr:`DEFAULT` special value to it. After deletion (or before first + assignment/load) attempting to read a property will get its default value + or, when no default, py:class:`exceptions.AttributeError`. + :param str name: name of the property :param collections.Callable setter: if not :py:obj:`None`, this is used to initialise value; first parameter to the function is holder instance and the second is value; this is called before ``type`` :param collections.Callable saver: function to coerce value to something readable by setter @@ -459,6 +464,10 @@ class property(object): ''' + #: Assigning this value to property means setting it to its default value. + #: If property has no default value, this will unset it. + DEFAULT = object() + def __init__(self, name, setter=None, saver=None, type=None, default=None, load_stage=2, order=0, save_via_ref=False, doc=None): self.__name__ = name @@ -498,6 +507,10 @@ class property(object): def __set__(self, instance, value): + if value is self.__class__.DEFAULT: + self.__delete__(instance) + return + try: oldvalue = getattr(instance, self.__name__) has_oldvalue = True diff --git a/qubes/tests/init.py b/qubes/tests/init.py index 78473bfa..45a990b3 100644 --- a/qubes/tests/init.py +++ b/qubes/tests/init.py @@ -123,7 +123,20 @@ class TC_10_property(qubes.tests.QubesTestCase): with self.assertRaises(AttributeError): self.holder.testprop - def test_091_delete_default(self): + def test_090_delete_by_assign(self): + self.holder.testprop1 = 'testvalue' + try: + if self.holder.testprop1 != 'testvalue': + self.skipTest('testprop1 value is wrong') + except AttributeError: + self.skipTest('testprop1 value is wrong') + + self.holder.testprop1 = qubes.property.DEFAULT + + with self.assertRaises(AttributeError): + self.holder.testprop + + def test_092_delete_default(self): class TestHolder(qubes.tests.TestEmitter, qubes.PropertyHolder): testprop1 = qubes.property('testprop1', default='defaultvalue') holder = TestHolder(None) From 2a62780ea2a55f2dbef93a8ec3ab5c25987322b8 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Thu, 8 Jan 2015 17:45:34 +0100 Subject: [PATCH 0039/1004] qubes: add property-del events --- qubes/__init__.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index 42553c7f..c27dbda9 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -527,7 +527,6 @@ class property(object): else: instance.fire_event_pre('property-pre-set:' + self.__name__, value) - instance._init_property(self, value) if has_oldvalue: @@ -537,8 +536,24 @@ class property(object): def __delete__(self, instance): + try: + oldvalue = getattr(instance, self.__name__) + has_oldvalue = True + except AttributeError: + has_oldvalue = False + + if has_oldvalue: + instance.fire_event_pre('property-pre-deleted:' + self.__name__, oldvalue) + else: + instance.fire_event_pre('property-pre-deleted:' + self.__name__) + delattr(instance, self._attr_name) + if has_oldvalue: + instance.fire_event('property-deleted:' + self.__name__, oldvalue) + else: + instance.fire_event('property-deleted:' + self.__name__) + def __repr__(self): return '<{} object at {:#x} name={!r} default={!r}>'.format( @@ -630,6 +645,22 @@ class PropertyHolder(qubes.events.Emitter): :param newvalue: New value of the property :param oldvalue: Old value of the property + .. event:: property-del: (subject, event, name[, oldvalue]) + + Fired when property gets deleted (is set to default). Signature is + variable, *oldvalue* is present only if there was an old value. + + :param name: Property name + :param oldvalue: Old value of the property + + .. event:: property-pre-del: (subject, event, name[, oldvalue]) + + Fired before property gets deleted (is set to default). Signature + is variable, *oldvalue* is present only if there was an old value. + + :param name: Property name + :param oldvalue: Old value of the property + Members: ''' From c2a35c02b48512f7e8bb37fc6d7c62b23e1bcf20 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Thu, 8 Jan 2015 19:13:51 +0100 Subject: [PATCH 0040/1004] qubes: Cache QubesHost requests, fix xen-specific members Acknowledgement: This commit is a result of core3 review by Marek. --- qubes/__init__.py | 90 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 16 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index c27dbda9..e4ce8e55 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -44,6 +44,7 @@ else: import libvirt try: import xen.lowlevel.xs + import xen.lowlevel.xc except ImportError: pass @@ -94,6 +95,8 @@ class VMMConnection(object): if 'xen.lowlevel.xs' in sys.modules: self._xs = xen.lowlevel.xs.xs() + if 'xen.lowlevel.cs' in sys.modules: + self._xc = xen.lowlevel.xc.xc() self._libvirt_conn = libvirt.open(defaults['libvirt_uri']) if self._libvirt_conn == None: raise QubesException("Failed connect to libvirt driver") @@ -110,10 +113,26 @@ class VMMConnection(object): def xs(self): '''Connection to Xen Store - This property in available only when running on Xen.''' + This property in available only when running on Xen. + ''' + # XXX what about the case when we run under KVM, but xen modules are importable? if 'xen.lowlevel.xs' not in sys.modules: - return None + raise AttributeError('xs object is available under Xen hypervisor only') + + self.init_vmm_connection() + return self._xs + + @__builtin__.property + def xc(self): + '''Connection to Xen + + This property in available only when running on Xen. + ''' + + # XXX what about the case when we run under KVM, but xen modules are importable? + if 'xen.lowlevel.xc' not in sys.modules: + raise AttributeError('xc object is available under Xen hypervisor only') self.init_vmm_connection() return self._xs @@ -122,44 +141,79 @@ class VMMConnection(object): class QubesHost(object): '''Basic information about host machine - :param Qubes app: Qubes application context (must have :py:attr:`Qubes.vmm` attribute defined) + :param qubes.Qubes app: Qubes application context (must have :py:attr:`Qubes.vmm` attribute defined) ''' def __init__(self, app): self._app = app + self._no_cpus = None + + + def _fetch(self): + if self._no_cpus is not None: + return (model, memory, cpus, mhz, nodes, socket, cores, threads) = \ self._app.vmm.libvirt_conn.getInfo() self._total_mem = long(memory)*1024 self._no_cpus = cpus -# print "QubesHost: total_mem = {0}B".format (self.xen_total_mem) -# print "QubesHost: free_mem = {0}".format (self.get_free_xen_memory()) -# print "QubesHost: total_cpus = {0}".format (self.xen_no_cpus) + self.app.log.debug('QubesHost: no_cpus={} memory_total={}'.format(self.no_cpus, self.memory_total)) + try: + self.app.log.debug('QubesHost: xen_free_memory={}'.format(self.get_free_xen_memory())) + except NotImplementedError: + pass + @__builtin__.property def memory_total(self): '''Total memory, in bytes''' + + self._fetch() return self._total_mem + @__builtin__.property def no_cpus(self): - '''Noumber of CPUs''' + '''Number of CPUs''' + + self._fetch() return self._no_cpus - # TODO - def get_free_xen_memory(self): - ret = self.physinfo['free_memory'] - return long(ret) - # TODO - def measure_cpu_usage(self, previous=None, previous_time = None, + def get_free_xen_memory(self): + '''Get free memory from Xen's physinfo. + + :raises NotImplementedError: when not under Xen + ''' + try: + self._physinfo = self.app.xc.physinfo() + except AttributeError: + raise NotImplementedError('This function requires Xen hypervisor') + return long(self._physinfo['free_memory']) + + + def measure_cpu_usage(self, previous_time=None, previous=None, wait_time=1): - """measure cpu usage for all domains at once""" + '''Measure cpu usage for all domains at once. + + This function requires Xen hypervisor. + + .. versionchanged:: 3.0 + argument order to match return tuple + + :raises NotImplementedError: when not under Xen + ''' + if previous is None: previous_time = time.time() previous = {} - info = self._app.vmm.xc.domain_getinfo(0, qubes_max_qid) + try: + info = self._app.vmm.xc.domain_getinfo(0, qubes_max_qid) + except AttributeError: + raise NotImplementedError( + 'This function requires Xen hypervisor') + for vm in info: previous[vm['domid']] = {} previous[vm['domid']]['cpu_time'] = ( @@ -169,7 +223,11 @@ class QubesHost(object): current_time = time.time() current = {} - info = self._app.vmm.xc.domain_getinfo(0, qubes_max_qid) + try: + info = self._app.vmm.xc.domain_getinfo(0, qubes_max_qid) + except AttributeError: + raise NotImplementedError( + 'This function requires Xen hypervisor') for vm in info: current[vm['domid']] = {} current[vm['domid']]['cpu_time'] = ( From 1deb3221c740c9b4e5530bc43e90fe180473667d Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Thu, 8 Jan 2015 19:35:59 +0100 Subject: [PATCH 0041/1004] qubes: fix netvm semantics WRT default values Automatic acquiring default*_netvm, default_template and {clock,update}vm is no more. This will be moved to firstboot. Advanced users (those, who elect not to autoconfig their initial VMs) will have to deal with that. Acknowledgement: This commit is a result of core3 review by Marek. --- qubes/__init__.py | 223 ++++++++++++++++++++++++++++++-------------- qubes/vm/qubesvm.py | 8 +- 2 files changed, 157 insertions(+), 74 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index e4ce8e55..91344203 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -389,7 +389,8 @@ class VMCollection(object): :raises ValueError: when there is already VM which has equal ``qid`` ''' - # XXX this violates duck typing, should we do it? + # this violates duck typing, but is needed + # for VMProperty to function correctly if not isinstance(value, qubes.vm.BaseVM): raise TypeError('{} holds only BaseVM instances'.format(self.__class__.__name__)) @@ -526,7 +527,10 @@ class property(object): #: If property has no default value, this will unset it. DEFAULT = object() - def __init__(self, name, setter=None, saver=None, type=None, default=None, + # internal use only + _NO_DEFAULT = object() + + def __init__(self, name, setter=None, saver=None, type=None, default=_NO_DEFAULT, load_stage=2, order=0, save_via_ref=False, doc=None): self.__name__ = name self._setter = setter @@ -556,7 +560,7 @@ class property(object): except AttributeError: # sys.stderr.write(' __get__ except\n') - if self._default is None: + if self._default is self._NO_DEFAULT: raise AttributeError('property {!r} not set'.format(self.__name__)) elif isinstance(self._default, collections.Callable): return self._default(instance) @@ -753,7 +757,44 @@ class PropertyHolder(qubes.events.Emitter): :param value: value ''' - setattr(self, prop._attr_name, value) + setattr(self, self.get_property_def(prop)._attr_name, value) + + + def property_is_default(self, prop): + '''Check whether property is in it's default value. + + Properties when unset may return some default value, so + ``hasattr(vm, prop.__name__)`` is wrong in some circumstances. This + method allows for checking if the value returned is in fact it's + default value. + + :param qubes.property prop: property object of particular interest + :rtype: bool + ''' + + return hasattr(self, self.get_property_def(prop)._attr_name) + + + def get_property_def(self, prop): + '''Return property definition object. + + If prop is already :py:class:`qubes.property` instance, return the same + object. + + :param prop: property object or name + :type prop: qubes.property or str + :rtype: qubes.property + ''' + + if isinstance(prop, qubes.property): + return prop + + for p in self.get_props_list(): + if p.__name__ == prop: + return p + + raise AttributeError('No property {!r} found in {!r}'.format( + prop, self.__class__)) def load_properties(self, load_stage=None): @@ -841,6 +882,30 @@ class PropertyHolder(qubes.events.Emitter): self.fire_event('cloned-properties', src, proplist) + def require_property(self, prop, allow_none=False, hard=False): + '''Complain badly when property is not set. + + :param prop: property name or object + :type prop: qubes.property or str + :param bool allow_none: if :py:obj:`True`, don't complain if :py:obj:`None` is found + :param bool hard: if :py:obj:`True`, raise :py:class:`AssertionError`; if :py:obj:`False`, log warning instead + ''' + + if isinstance(qubes.property, prop): + prop = prop.__name__ + + try: + value = getattr(self, prop) + if value is None and not allow_none: + raise AttributeError() + except AttributeError: + msg = 'Required property {!r} not set on {!r}'.format(prop, self) + if hard: + raise AssertionError(msg) + else: + self.log(msg) + + import qubes.vm @@ -852,19 +917,34 @@ class VMProperty(property): and all supported by :py:class:`property` with the exception of ``type`` and ``setter`` ''' - def __init__(self, name, vmclass=qubes.vm.BaseVM, **kwargs): + def __init__(self, name, vmclass=qubes.vm.BaseVM, allow_none=False, **kwargs): if 'type' in kwargs: raise TypeError("'type' keyword parameter is unsupported in {}".format( self.__class__.__name__)) if 'setter' in kwargs: raise TypeError("'setter' keyword parameter is unsupported in {}".format( self.__class__.__name__)) + if not issubclass(vmclass, qubes.vm.BaseVM): + raise TypeError("'vmclass' should specify a subclass of qubes.vm.BaseVM") + super(VMProperty, self).__init__(name, **kwargs) self.vmclass = vmclass - + self.allow_none = allow_none def __set__(self, instance, value): + if value is None: + if self.allow_none: + super(VMProperty, self).__set__(self, instance, vm) + return + else: + raise ValueError( + 'Property {!r} does not allow setting to {!r}'.format( + self.__name__, value)) + + # XXX this may throw LookupError; that's good until introduction + # of QubesNoSuchVMException or whatever vm = instance.app.domains[value] + if not isinstance(vm, self.vmclass): raise TypeError('wrong VM class: domains[{!r}] if of type {!s} and not {!s}'.format( value, vm.__class__.__name__, self.vmclass.__name__)) @@ -875,7 +955,6 @@ class VMProperty(property): import qubes.vm.qubesvm import qubes.vm.templatevm - class Qubes(PropertyHolder): '''Main Qubes application @@ -922,10 +1001,13 @@ class Qubes(PropertyHolder): Methods and attributes: ''' - default_netvm = VMProperty('default_netvm', load_stage=3, - doc='Default NetVM for new AppVMs') - default_fw_netvm = VMProperty('default_fw_netvm', load_stage=3, - doc='Default NetVM for new ProxyVMs') + default_netvm = VMProperty('default_netvm', load_stage=3, default=None, + doc='''Default NetVM for AppVMs. Initial state is :py:obj:`None`, which + means that AppVMs are not connected to the Internet.''') + default_fw_netvm = VMProperty('default_fw_netvm', load_stage=3, default=None, + doc='''Default NetVM for ProxyVMs. Initial state is :py:obj:`None`, which + means that ProxyVMs (including FirewallVM) are not connected to the + Internet.''') default_template = VMProperty('default_template', load_stage=3, vmclass=qubes.vm.templatevm.TemplateVM, doc='Default template for new AppVMs') @@ -1006,40 +1088,22 @@ class Qubes(PropertyHolder): # stage 5: misc fixups - # if we have no default netvm, make first one the default - if not hasattr(self, 'default_netvm'): - for vm in self.domains: - if hasattr(vm, 'provides_network') and hasattr(vm, 'netvm'): - self.default_netvm = vm - break - - if not hasattr(self, 'default_fw_netvm'): - for vm in self.domains: - if hasattr(vm, 'provides_network') and not hasattr(vm, 'netvm'): - self.default_netvm = vm - break - - # first found template vm is the default - if not hasattr(self, 'default_template'): - for vm in self.domains: - if isinstance(vm, qubes.vm.templatevm.TemplateVM): - self.default_template = vm - break - - # if there was no clockvm entry in qubes.xml, try to determine default: - # root of default NetVM chain - if not hasattr(self, 'clockvm') and hasattr(self, 'default_netvm'): - clockvm = self.default_netvm - # Find root of netvm chain - while clockvm.netvm is not None: - clockvm = clockvm.netvm - - self.clockvm = clockvm + self.require_property('default_fw_netvm', allow_none=True) + self.require_property('default_netvm', allow_none=True) + self.require_property('default_template') + self.require_property('clockvm') + self.require_property('updatevm') # Disable ntpd in ClockVM - to not conflict with ntpdate (both are # using 123/udp port) if hasattr(self, 'clockvm'): - self.clockvm.services['ntpd'] = False + if 'ntpd' in self.clockvm.services: + if self.clockvm.services['ntpd']: + self.log.warning("VM set as clockvm ({!r}) has enabled " + "'ntpd' service! Expect failure when syncing time in " + "dom0.".format(self.clockvm)) + else: + self.clockvm.services['ntpd'] = False def _init(self): @@ -1102,7 +1166,6 @@ class Qubes(PropertyHolder): return labels - def add_new_vm(self, vm): '''Add new Virtual Machine to colletion @@ -1110,35 +1173,16 @@ class Qubes(PropertyHolder): self.domains.add(vm) - @qubes.events.handler('domain-added') - def on_domain_addedd(self, event, vm): - # make first created NetVM the default one - if not hasattr(self, 'default_fw_netvm') \ - and vm.provides_network \ - and not hasattr(vm, 'netvm'): - self.default_fw_netvm = vm - if not hasattr(self, 'default_netvm') \ - and vm.provides_network \ - and hasattr(vm, 'netvm'): - self.default_netvm = vm - - # make first created TemplateVM the default one - if not hasattr(self, 'default_template') \ - and not hasattr(vm, 'template'): - self.default_template = vm - - # make first created ProxyVM the UpdateVM - if not hasattr(self, 'default_netvm') \ - and vm.provides_network \ - and hasattr(vm, 'netvm'): - self.updatevm = vm - - # by default ClockVM is the first NetVM - if not hasattr(self, 'clockvm') \ - and vm.provides_network \ - and hasattr(vm, 'netvm'): - self.default_clockvm = vm + @qubes.events.handler('domain-pre-deleted') + def on_domain_pre_deleted(self, event, vm): + if isinstance(vm, qubes.vm.templatevm.TemplateVM): + appvms = self.get_vms_based_on(vm) + if appvms: + raise QubesException( + 'Cannot remove template that has dependent AppVMs. ' + 'Affected are: {}'.format(', '.join( + vm.name for name in sorted(appvms)))) @qubes.events.handler('domain-deleted') @@ -1157,5 +1201,44 @@ class Qubes(PropertyHolder): return super(QubesVmCollection, self).pop(qid) + @qubes.events.handler('property-pre-set:clockvm') + def on_property_pre_set_clockvm(self, event, name, newvalue, oldvalue=None): + if 'ntpd' in newvalue.services: + if newvalue.services['ntpd']: + raise QubesException( + 'Cannot set {!r} as {!r} property since it has ntpd enabled.'.format( + newvalue, name)) + else: + newvalue.services['ntpd'] = False + + + @qubes.events.handler('property-pre-set:default_netvm') + def on_property_pre_set_default_netvm(self, event, name, newvalue, oldvalue=None): + if newvalue is not None and oldvalue is not None \ + and oldvalue.is_running() and not newvalue.is_running() \ + and self.domains.get_vms_connected_to(oldvalue): + raise QubesException( + 'Cannot change default_netvm to domain that is not running ({!r}).'.format( + newvalue)) + + + @qubes.events.handler('property-set:default_fw_netvm') + def on_property_set_default_netvm(self, event, name, newvalue, oldvalue=None): + for vm in self.domains: + if not vm.provides_network and vm.property_is_default('netvm'): + # fire property-del:netvm as it is responsible for resetting + # netvm to it's default value + vm.fire_event('property-del:netvm', 'netvm', newvalue, oldvalue) + + + @qubes.events.handler('property-set:default_netvm') + def on_property_set_default_netvm(self, event, name, newvalue, oldvalue=None): + for vm in self.domains: + if vm.provides_network and vm.property_is_default('netvm'): + # fire property-del:netvm as it is responsible for resetting + # netvm to it's default value + vm.fire_event('property-del:netvm', 'netvm', newvalue, oldvalue) + + # load plugins import qubes._pluginloader diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 9c9bc24d..be2d9765 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -111,12 +111,12 @@ class QubesVM(qubes.vm.BaseVM): 'of the padlock.') # XXX swallowed uses_default_netvm - netvm = qubes.property('netvm', load_stage=4, + netvm = qubes.VMProperty('netvm', load_stage=4, allow_none=True, default=(lambda self: self.app.default_fw_netvm if self.provides_network else self.app.default_netvm), doc='VM that provides network connection to this domain. ' - 'When :py:obj:`False`, machine is disconnected. ' - 'When :py:obj:`None` (or absent), domain uses default NetVM.') + 'When :py:obj:`None`, machine is disconnected. ' + 'When absent, domain uses default NetVM.') provides_network = qubes.property('provides_network', type=bool, doc=':py:obj:`True` if it is NetVM or ProxyVM, false otherwise') @@ -465,7 +465,7 @@ class QubesVM(qubes.vm.BaseVM): # we are changing to default netvm new_netvm = self.netvm if new_netvm == old_netvm: return - self.on_property_set_netvm(self, event, name, new_netvm, old_netvm) + self.fire_event('property-set:netvm', 'netvm', new_netvm, old_netvm) @qubes.events.handler('property-set:netvm') From c0e3281d04b1685986a17608a53f636f0258aab1 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Thu, 8 Jan 2015 19:44:14 +0100 Subject: [PATCH 0042/1004] qubes: fix changing domain name Changing name of running VM is wrong. Acknowledgement: This commit is a result of core3 review by Marek. --- qubes/vm/qubesvm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index be2d9765..e69b12d1 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -509,6 +509,9 @@ class QubesVM(qubes.vm.BaseVM): @qubes.events.handler('property-set:name') def on_property_set_name(self, event, name, new_name, old_name=None): + # TODO not self.is_stopped() would be more appropriate + if self.is_running(): + raise QubesException('Cannot change name of running domain') if self.libvirt_domain: self.libvirt_domain.undefine() self._libvirt_domain = None From 7971342811c3d9ea734a6efd1e4a436381e6cdd7 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Fri, 9 Jan 2015 15:09:56 +0100 Subject: [PATCH 0043/1004] qubes: Make get_props_list a classmethod Same for get_property_def. --- qubes/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index 91344203..c9491388 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -731,8 +731,9 @@ class PropertyHolder(qubes.events.Emitter): self.xml = xml - def get_props_list(self, load_stage=None): - '''List all properties attached to this VM + @classmethod + def get_props_list(cls, load_stage=None): + '''List all properties attached to this VM's class :param load_stage: Filter by load stage :type load_stage: :py:func:`int` or :py:obj:`None` @@ -740,7 +741,7 @@ class PropertyHolder(qubes.events.Emitter): # sys.stderr.write('{!r}.get_props_list(load_stage={})\n'.format('self', load_stage)) props = set() - for class_ in self.__class__.__mro__: + for class_ in cls.__mro__: props.update(prop for prop in class_.__dict__.values() if isinstance(prop, property)) if load_stage is not None: @@ -775,7 +776,8 @@ class PropertyHolder(qubes.events.Emitter): return hasattr(self, self.get_property_def(prop)._attr_name) - def get_property_def(self, prop): + @classmethod + def get_property_def(cls, prop): '''Return property definition object. If prop is already :py:class:`qubes.property` instance, return the same @@ -789,12 +791,12 @@ class PropertyHolder(qubes.events.Emitter): if isinstance(prop, qubes.property): return prop - for p in self.get_props_list(): + for p in cls.get_props_list(): if p.__name__ == prop: return p raise AttributeError('No property {!r} found in {!r}'.format( - prop, self.__class__)) + prop, cls)) def load_properties(self, load_stage=None): From 99edcb56c1db4cd6e485a0f8dfabb530a565a6ca Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 12 Jan 2015 14:57:24 +0100 Subject: [PATCH 0044/1004] qubes: fix event framework Two important fixes are in this commit: handlers from decorators are added when class is defined (and not when class is instantiated); also multiple events can be specified in the decorator. --- qubes/events.py | 51 +++++++++++++++++++------------------------ qubes/ext/__init__.py | 12 +++++----- 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/qubes/events.py b/qubes/events.py index 58180f59..4eb03361 100644 --- a/qubes/events.py +++ b/qubes/events.py @@ -10,7 +10,7 @@ etc. import collections -def handler(event): +def handler(*events): '''Event handler decorator factory. To hook an event, decorate a method in your plugin class with this @@ -27,8 +27,7 @@ def handler(event): ''' def decorator(f): - f.ha_event = event - f.ha_bound = True + f.ha_events = events return f return decorator @@ -43,7 +42,7 @@ def ishandler(o): ''' return callable(o) \ - and hasattr(o, 'ha_event') + and hasattr(o, 'ha_events') class EmitterMeta(type): @@ -52,6 +51,24 @@ class EmitterMeta(type): super(type, cls).__init__(name, bases, dict_) cls.__handlers__ = collections.defaultdict(set) + try: + propnames = set(prop.__name__ for prop in cls.get_props_list()) + except AttributeError: + propnames = set() + + for attr in dict_: + if attr in propnames: + # we have to be careful, not to getattr() on properties which + # may be unset + continue + + attr = dict_[attr] + if not ishandler(attr): + continue + + for event in attr.ha_events: + cls.add_handler(event, attr) + class Emitter(object): '''Subject that can emit events @@ -63,23 +80,6 @@ class Emitter(object): super(Emitter, self).__init__(*args, **kwargs) self.events_enabled = True - try: - propnames = set(prop.__name__ for prop in self.get_props_list()) - except AttributeError: - propnames = set() - - for attr in dir(self): - if attr in propnames: - # we have to be careful, not to getattr() on properties which - # may be unset - continue - - attr = getattr(self, attr) - if not ishandler(attr): - continue - - self.add_handler(attr.ha_event, attr) - @classmethod def add_handler(cls, event, handler): @@ -103,18 +103,11 @@ class Emitter(object): return for cls in order: - # first fire bound (= our own) handlers, then handlers from extensions if not hasattr(cls, '__handlers__'): continue for handler in sorted(cls.__handlers__[event], key=(lambda handler: hasattr(handler, 'ha_bound')), reverse=True): - if hasattr(handler, 'ha_bound'): - # this is our (bound) method, self is implicit - handler(event, *args, **kwargs) - else: - # this is from extension or hand-added, so we see method as - # unbound, therefore we need to pass self - handler(self, event, *args, **kwargs) + handler(self, event, *args, **kwargs) def fire_event(self, event, *args, **kwargs): diff --git a/qubes/ext/__init__.py b/qubes/ext/__init__.py index b9c1e469..1a98e339 100644 --- a/qubes/ext/__init__.py +++ b/qubes/ext/__init__.py @@ -55,7 +55,7 @@ class Extension(object): self.app.add_hook(attr.ha_event, attr) -def handler(event, vm=None, system=False): +def handler(*events, **kwargs): '''Event handler decorator factory. To hook an event, decorate a method in your plugin class with this @@ -71,14 +71,14 @@ def handler(event, vm=None, system=False): ''' def decorator(f): - f.ho_event = event + f.ha_events = events - if system: + if kwargs.get('system', False): f.ha_vm = None - elif vm is None: - f.ha_vm = qubes.vm.BaseVM + elif 'vm' in kwargs: + f.ha_vm = kwargs['vm'] else: - f.ha_vm = vm + f.ha_vm = qubes.vm.BaseVM return f From 0a94762508aa8c9923b994fd82ef1c1edb3ea1d6 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 12 Jan 2015 15:48:17 +0100 Subject: [PATCH 0045/1004] doc: Tutorial for qubes.events and fix --- doc/qubes-events.rst | 124 +++++++++++++++++++++++++++++++++++++++++++ qubes/dochelpers.py | 2 +- 2 files changed, 125 insertions(+), 1 deletion(-) diff --git a/doc/qubes-events.rst b/doc/qubes-events.rst index 636042ef..fe011ebb 100644 --- a/doc/qubes-events.rst +++ b/doc/qubes-events.rst @@ -1,6 +1,130 @@ :py:mod:`qubes.events` -- Qubes events ====================================== +Some objects in qubes (most notably domains) emit events. You may hook them and +execute your code when particular event is fired. Events in qubes are added +class-wide -- it is not possible to add event handler to one instance only, you +have to add handler for whole class. + + +Firing events +------------- + +Events are fired by calling :py:meth:`qubes.events.Emitter.fire_event`. The +first argument is event name (a string). You can fire any event you wish, the +names are not checked in any way, however each class' documentation tells what +standard events will be fired on it. The rest of arguments are dependent on the +particular event in question -- they are passed as-is to handlers. + +Event handlers are fired in reverse method resolution order, that is, first for +parent class and then for it's child. For each class, first are called handlers +defined in it's source, then handlers from extensions and last the callers added +manually. + +There is second method, :py:meth:`qubes.events.Emitter.fire_event_pre`, which +fires events in reverse order. It is suitable for events fired before some +action is performed. You may at your own responsibility raise exceptions from +such events to try to prevent such action. + + +Handling events +--------------- + +There are several ways to handle events. In all cases you supply a callable +(most likely function or method) that will be called when someone fires the +event. The first argument passed to the callable will be the object instance on +which the event was fired and the second one is the event name. The rest are +passed from :py:meth:`qubes.events.Emitter.fire_event` as described previously. +One callable can handle more than one event. + +The easiest way to hook an event is to invoke +:py:meth:`qubes.events.Emitter.add_handler` classmethod. + +.. code-block:: python + + import qubes.events + + class MyClass(qubes.events.Emitter): + pass + + def event_handler(subject, event): + if event == 'event1': + print('Got event 1') + elif event == 'event2': + print('Got event 2') + + MyClass.add_handler('event1', event_handler) + MyClass.add_handler('event2', event_handler) + + o = MyClass() + o.fire_event('event1') + +If you wish to define handler in the class definition, the best way is to use +:py:func:`qubes.events.handler` decorator. + +.. code-block:: python + + import qubes.events + + class MyClass(qubes.events.Emitter): + @qubes.events.handler('event1', 'event2') + def event_handler(self, event): + if event == 'event1': + print('Got event 1') + elif event == 'event2': + print('Got event 2') + + o = MyClass() + o.fire_event('event1') + +.. TODO: extensions + + +Handling events with variable signature +--------------------------------------- + +Some events are specified with variable signature (i.e. they may have different +number of arguments on each call to handlers). You can write handlers just like +every other python function with variable signature. + +.. code-block:: python + + import qubes + + def on_property_change(subject, event, name, newvalue, oldvalue=None): + if oldvalue is None: + print('Property {} initialised to {!r}'.format(name, newvalue)) + else: + print('Property {} changed {!r} -> {!r}'.format(name, oldvalue, newvalue)) + + qubes.Qubes.add_handler('property-set:default_netvm') + +If you expect :py:obj:`None` to be a reasonable value of the property, you have +a problem. One way to solve it is to invent your very own, magic +:py:class:`object` instance. + +.. code-block:: python + + import qubes + + MAGIC_NO_VALUE = object() + def on_property_change(subject, event, name, newvalue, oldvalue=MAGIC_NO_VALUE): + if oldvalue is MAGIC_NO_VALUE: + print('Property {} initialised to {!r}'.format(name, newvalue)) + else: + print('Property {} changed {!r} -> {!r}'.format(name, oldvalue, newvalue)) + + qubes.Qubes.add_handler('property-set:default_netvm') + +There is no possible way of collision other than intentionally passing this very +object (not even passing similar featureless ``object()``), because ``is`` +python syntax checks object's :py:meth:`id`\ entity, which will be different for +each :py:class:`object` instance. + + +Module contents +--------------- + .. automodule:: qubes.events :members: :show-inheritance: diff --git a/qubes/dochelpers.py b/qubes/dochelpers.py index 4c5a568c..f8abf3ab 100644 --- a/qubes/dochelpers.py +++ b/qubes/dochelpers.py @@ -123,7 +123,7 @@ class VersionCheck(docutils.parsers.rst.Directive): # this is lifted from sphinx' own conf.py # -event_sig_re = re.compile(r'([a-zA-Z-]+)\s*\((.*)\)') +event_sig_re = re.compile(r'([a-zA-Z-:<>]+)\s*\((.*)\)') def parse_event(env, sig, signode): m = event_sig_re.match(sig) From 091ffa544484855af53939f298efb912267ac644 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 12 Jan 2015 16:56:14 +0100 Subject: [PATCH 0046/1004] qubes: Add parser for property docstring From now, docstrings in properties cannot contain sphinx-specific features, because there is no sphinx in dom0. --- qubes/__init__.py | 35 +++++++++++++++++++++++++++++------ qubes/vm/qubesvm.py | 31 +++++++++++++++++-------------- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index c9491388..4f148d44 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -26,6 +26,8 @@ import warnings import __builtin__ +import docutils.core +import docutils.io import lxml.etree import xml.parsers.expat @@ -501,7 +503,7 @@ class property(object): :param object default: default value; if callable, will be called with holder as first argument :param int load_stage: stage when property should be loaded (see :py:class:`Qubes` for description of stages) :param int order: order of evaluation (bigger order values are later) - :param str doc: docstring; you may use RST markup + :param str doc: docstring; this should be one paragraph of plain RST, no sphinx-specific features Setters and savers have following signatures: @@ -630,6 +632,26 @@ class property(object): return self.__name__ == other.__name__ + def format_doc(self): + '''Return parsed documentation string, stripping RST markup. + ''' + + if not self.__doc__: return '' + + output, pub = docutils.core.publish_programmatically( + source_class=docutils.io.StringInput, + source=' '.join(self.__doc__.strip().split()), + source_path=None, + destination_class=docutils.io.NullOutput, destination=None, + destination_path=None, + reader=None, reader_name='standalone', + parser=None, parser_name='restructuredtext', + writer=None, writer_name='null', + settings=None, settings_spec=None, settings_overrides=None, + config_section=None, enable_exit_status=None) + return pub.writer.document.astext() + + # # exceptions # @@ -1004,17 +1026,18 @@ class Qubes(PropertyHolder): ''' default_netvm = VMProperty('default_netvm', load_stage=3, default=None, - doc='''Default NetVM for AppVMs. Initial state is :py:obj:`None`, which - means that AppVMs are not connected to the Internet.''') + doc='''Default NetVM for AppVMs. Initial state is `None`, which means + that AppVMs are not connected to the Internet.''') default_fw_netvm = VMProperty('default_fw_netvm', load_stage=3, default=None, - doc='''Default NetVM for ProxyVMs. Initial state is :py:obj:`None`, which - means that ProxyVMs (including FirewallVM) are not connected to the + doc='''Default NetVM for ProxyVMs. Initial state is `None`, which means + that ProxyVMs (including FirewallVM) are not connected to the Internet.''') default_template = VMProperty('default_template', load_stage=3, vmclass=qubes.vm.templatevm.TemplateVM, doc='Default template for new AppVMs') updatevm = VMProperty('updatevm', load_stage=3, - doc='Which VM to use as ``yum`` proxy for updating AdminVM and TemplateVMs') + doc='''Which VM to use as `yum` proxy for updating AdminVM and + TemplateVMs''') clockvm = VMProperty('clockvm', load_stage=3, doc='Which VM to use as NTP proxy for updating AdminVM') default_kernel = property('default_kernel', load_stage=3, diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index e69b12d1..d9b46dad 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -114,17 +114,17 @@ class QubesVM(qubes.vm.BaseVM): netvm = qubes.VMProperty('netvm', load_stage=4, allow_none=True, default=(lambda self: self.app.default_fw_netvm if self.provides_network else self.app.default_netvm), - doc='VM that provides network connection to this domain. ' - 'When :py:obj:`None`, machine is disconnected. ' - 'When absent, domain uses default NetVM.') + doc='''VM that provides network connection to this domain. When + `None`, machine is disconnected. When absent, domain uses default + NetVM.''') provides_network = qubes.property('provides_network', type=bool, - doc=':py:obj:`True` if it is NetVM or ProxyVM, false otherwise') + doc='`True` if it is NetVM or ProxyVM, false otherwise.') qid = qubes.property('qid', type=int, setter=_setter_qid, - doc='Internal, persistent identificator of particular domain. ' - 'Note this is different from Xen domid.') + doc='''Internal, persistent identificator of particular domain. Note + this is different from Xen domid.''') name = qubes.property('name', type=str, doc='User-specified name of the domain.') @@ -146,19 +146,20 @@ class QubesVM(qubes.vm.BaseVM): installed_by_rpm = qubes.property('installed_by_rpm', type=bool, default=False, setter=qubes.property.bool, - doc="If this domain's image was installed from package tracked by " - "package manager.") + doc='''If this domain's image was installed from package tracked by + package manager.''') memory = qubes.property('memory', type=int, default=qubes.config.defaults['memory'], doc='Memory currently available for this VM.') maxmem = qubes.property('maxmem', type=int, default=None, - doc='Maximum amount of memory available for this VM ' - '(for the purpose of memory balancer).') + doc='''Maximum amount of memory available for this VM (for the purpose + of the memory balancer).''') internal = qubes.property('internal', type=bool, default=False, setter=qubes.property.bool, - doc="Internal VM (not shown in qubes-manager, doesn't create appmenus entries.") + doc='''Internal VM (not shown in qubes-manager, don't create appmenus + entries.''') # XXX what is that vcpus = qubes.property('vcpus', default=None, @@ -201,12 +202,14 @@ class QubesVM(qubes.vm.BaseVM): # return self._default_user qrexec_timeout = qubes.property('qrexec_timeout', type=int, default=60, - doc='Time in seconds after which qrexec connection attempt is deemed failed. ' - 'Operating system inside VM should be able to boot in this time.') + doc='''Time in seconds after which qrexec connection attempt is deemed + failed. Operating system inside VM should be able to boot in this + time.''') autostart = qubes.property('autostart', type=bool, default=False, setter=qubes.property.bool, - doc='Setting this to :py:obj:`True` means that VM should be autostarted on dom0 boot.') + doc='''Setting this to `True` means that VM should be autostarted on dom0 + boot.''') # XXX I don't understand backups include_in_backups = qubes.property('include_in_backups', type=bool, default=True, From 118edb6ac476c3c8f04909418bd86bbb8f6e67b4 Mon Sep 17 00:00:00 2001 From: Wojciech Zygmunt Porczyk Date: Sun, 11 Jan 2015 01:19:03 +0100 Subject: [PATCH 0047/1004] First RelaxNG schema of qubes.xml Schema is most likely incomplete. --- qubes/tests/__init__.py | 43 ++++++ qubes/tests/vm/init.py | 2 +- relaxng/domain.rng | 12 ++ relaxng/qubes.rng | 308 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 relaxng/domain.rng create mode 100644 relaxng/qubes.rng diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 00721fcc..d1b63f20 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -3,6 +3,8 @@ import collections import unittest +import lxml.etree + import qubes.config import qubes.events @@ -132,3 +134,44 @@ class QubesTestCase(unittest.TestCase): self.fail('event {!r} did fire on {!r}'.format(event, emitter)) return + + + def assertXMLIsValid(self, xml, file=None, schema=None): + '''Check whether given XML fulfills Relax NG schema. + + Schema can be given in a couple of ways: + + - As separate file. This is most common, and also the only way to + handle file inclusion. Call with file name as second argument. + + - As string containing actual schema. Put that string in *schema* + keyword argument. + + :param lxml.etree._Element xml: XML element instance to check + :param str file: filename of Relax NG schema + :param str schema: optional explicit schema string + ''' + + if schema is not None and file is None: + relaxng = schema + if isinstance(relaxng, str): + relaxng = lxml.etree.XML(relaxng) + if isinstance(relaxng, lxml.etree._Element): + relaxng = lxml.etree.RelaxNG(relaxng) + + elif file is not None and schema is None: + relaxng = lxml.etree.RelaxNG(file=file) + + else: + raise TypeError("There should be excactly one of 'file' and " + "'schema' arguments specified.") + + # We have to be extra careful here in case someone messed up with + # self.failureException. It should by default be AssertionError, just + # what is spewed by RelaxNG(), but who knows what might happen. + try: + relaxng.assert_(xml) + except self.failureException: + raise + except AssertionError as e: + self.fail(str(e)) diff --git a/qubes/tests/vm/init.py b/qubes/tests/vm/init.py index 1608f189..06b28374 100644 --- a/qubes/tests/vm/init.py +++ b/qubes/tests/vm/init.py @@ -131,7 +131,7 @@ class TC_10_BaseVM(qubes.tests.QubesTestCase): 'disabledservice': False, }) - lxml.etree.ElementTree(vm.__xml__()).write(sys.stderr, encoding='utf-8', pretty_print=True) + self.assertXMLIsValid(vm.__xml__(), '../../relaxng/domain.rng') def test_001_BaseVM_nxproperty(self): xml = lxml.etree.XML(''' diff --git a/relaxng/domain.rng b/relaxng/domain.rng new file mode 100644 index 00000000..3305db34 --- /dev/null +++ b/relaxng/domain.rng @@ -0,0 +1,12 @@ + + + + + + + + + + diff --git a/relaxng/qubes.rng b/relaxng/qubes.rng new file mode 100644 index 00000000..82a38a1c --- /dev/null +++ b/relaxng/qubes.rng @@ -0,0 +1,308 @@ + + + + + + + + + + + + This is root element of whole qubes tree. + + + + + Specifies minimal Qubes OS version. + + + 3.0 + + + + + + + + Container for labels. + + + + + + Label which can be used by domain. One choice of + colour for padlock icon. + + + + + XML id attribute used for cross-referencing in + properties' ``ref`` attribute. + + + + + label-[0-9]+ + + + + + + + Label's colour, HTML-like. + + + + #[0-9a-f]{6} + + + + + + [a-z0-9_-]+ + + + + + + + + Container for domains. + + + + + + + + + + + + + One Qubes domain. + + + + + Type of the domain. This specifies Python's class that is + used for instantiation of this VM. + + + + + + + XML id attribute used for cross-referencing in properties' + ``ref`` attribute. + + + + + domain-[0-9]+ + + + + + + + + + Container for services. + + + + + + One service that is either enabled or disabled. + + + + + + Whether service is enabled or disabled. + Default is ``true``. + + + true|false + + + + + + + + [a-z0-9_-]+ + + + + + + + + + + Container for devices of particular class. + + + + + Class of devices in this container. Currently the + only supported is ``pci``. + + + + + pci + + + + + + One device. This tag should contain some + identifier, format of which depends on + particular device class. + + + + [0-9a-f]{2}:[0-9a-f]{2}.[0-9a-f]{2} + + + + + + + + + + Container for user-defined tags. + + + + + + Tag value. + + Tags are not used anywhere by qubes core, they + are for users reference. In the future they + will be available for use in policies. + + + + + Name of the tag. + + + + [a-z0-9_-]+ + + + + + + + + + + + + + + + + + Container for properties. + + + + + + One property and its value specified either directly + (as text contained in this tag) or as reference to + another XML element in the tree (by ``ref=`` + attribute). How it is saved, it depends on particular + property. + + + + + Property name. + + + + [a-z0-9_]+ + + + + + + + Alternative property value, by reference to another XML element. + + + + + + + + + + + + + + + From 6df316378b9df50c474a75a5441bec3f6e472758 Mon Sep 17 00:00:00 2001 From: Wojciech Zygmunt Porczyk Date: Mon, 12 Jan 2015 12:33:18 +0100 Subject: [PATCH 0048/1004] doc: Fix docutils table column width --- doc/_static/qubes.css | 9 +++++++++ doc/_templates/layout.html | 4 ++++ 2 files changed, 13 insertions(+) create mode 100644 doc/_static/qubes.css create mode 100644 doc/_templates/layout.html diff --git a/doc/_static/qubes.css b/doc/_static/qubes.css new file mode 100644 index 00000000..3e0078d5 --- /dev/null +++ b/doc/_static/qubes.css @@ -0,0 +1,9 @@ +/* http://stackoverflow.com/questions/21027105/how-can-i-have-sphinx-tables-fit-to-width */ + +table.docutils col { + width: auto; +} + +/* +vim: ts=4 sw=4 et +*/ diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html new file mode 100644 index 00000000..efbcf4b7 --- /dev/null +++ b/doc/_templates/layout.html @@ -0,0 +1,4 @@ +{% extends '!layout.html' %} +{% set css_files = css_files + ['_static/qubes.css'] %} + +{# vim: ts=4 sw=4 et #} From 42d8e67556f6b7eacf711be439b1992b3df07269 Mon Sep 17 00:00:00 2001 From: Wojciech Zygmunt Porczyk Date: Mon, 12 Jan 2015 13:22:04 +0100 Subject: [PATCH 0049/1004] doc: Add autogenerated qubes.xml documentation --- doc/.gitignore | 2 + doc/Makefile | 45 ++++++----- doc/example.xml | 45 +++++++++++ doc/index.rst | 5 ++ qubes/rngdoc.py | 182 ++++++++++++++++++++++++++++++++++++++++++++ qubes/tests/init.py | 5 +- relaxng/qubes.rng | 3 - 7 files changed, 263 insertions(+), 24 deletions(-) create mode 100644 doc/example.xml create mode 100755 qubes/rngdoc.py diff --git a/doc/.gitignore b/doc/.gitignore index 512245fb..63b63ff9 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -1 +1,3 @@ +_build sandbox.rst +autoxml.rst diff --git a/doc/Makefile b/doc/Makefile index 95b72af8..dae418dd 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -14,6 +14,8 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +DEPEND = autoxml.rst + .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @@ -41,40 +43,40 @@ help: @echo " install to generate manpages and copy them to \$$(DESTDIR)/usr/share/man" clean: - -rm -rf $(BUILDDIR)/* + -rm -rf $(BUILDDIR)/* $(DEPEND) -html: +html: $(DEPEND) $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." -dirhtml: +dirhtml: $(DEPEND) $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." -singlehtml: +singlehtml: $(DEPEND) $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." -pickle: +pickle: $(DEPEND) $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." -json: +json: $(DEPEND) $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." -htmlhelp: +htmlhelp: $(DEPEND) $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." -qthelp: +qthelp: $(DEPEND) $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -83,7 +85,7 @@ qthelp: @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/core-admin.qhc" -devhelp: +devhelp: $(DEPEND) $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @@ -92,30 +94,30 @@ devhelp: @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/core-admin" @echo "# devhelp" -epub: +epub: $(DEPEND) $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." -latex: +latex: $(DEPEND) $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." -latexpdf: +latexpdf: $(DEPEND) $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." -text: +text: $(DEPEND) $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." -man: +man: $(DEPEND) $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man for file in $(BUILDDIR)/man/*.[12345678]; do \ gzip -f $$file; \ @@ -123,41 +125,44 @@ man: @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." -texinfo: +texinfo: $(DEPEND) $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." -info: +info: $(DEPEND) $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." -gettext: +gettext: $(DEPEND) $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." -changes: +changes: $(DEPEND) $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." -linkcheck: +linkcheck: $(DEPEND) $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." -doctest: +doctest: $(DEPEND) $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." +autoxml.rst: ../relaxng/qubes.rng example.xml + ../qubes/rngdoc.py $+ > $@ + .PHONY: install install: man mkdir -p $(DESTDIR)/usr/share/man/man1 diff --git a/doc/example.xml b/doc/example.xml new file mode 100644 index 00000000..0d1b5756 --- /dev/null +++ b/doc/example.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + 1 + netvm + + + + + meminfo-writer + qubes-firewall + + + + 01:23.45 + + + + + + 2 + appvm + + + + + qwe123 + + + + + + diff --git a/doc/index.rst b/doc/index.rst index 9552e67e..cb3de45a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -35,6 +35,11 @@ Developer documentation qubes-tests qubes-dochelpers +.. toctree:: + :maxdepth: 1 + + autoxml + Indices and tables ================== diff --git a/qubes/rngdoc.py b/qubes/rngdoc.py new file mode 100755 index 00000000..8467d68a --- /dev/null +++ b/qubes/rngdoc.py @@ -0,0 +1,182 @@ +#!/usr/bin/python2 -O + +from __future__ import print_function + +import sys +import textwrap + +import lxml.etree + +class Element(object): + def __init__(self, schema, xml): + self.schema = schema + self.xml = xml + + + @property + def nsmap(self): + return self.schema.nsmap + + + @property + def name(self): + return self.xml.get('name') + + + def get_description(self, xml=None, wrap=True): + if xml is None: xml = self.xml + + xml = xml.xpath('./doc:description', namespaces=self.nsmap) + if not xml: return '' + xml = xml[0] + + if wrap: + return ''.join(self.schema.wrapper.fill(p) + '\n\n' + for p in textwrap.dedent(xml.text.strip('\n')).split('\n\n')) + else: + return ' '.join(xml.text.strip().split()) + + + def get_data_type(self, xml=None): + if xml is None: xml = self.xml + + value = xml.xpath('./rng:value', namespaces=self.nsmap) + if value: + value = '``{}``'.format(value[0].text.strip()) + else: + metavar = xml.xpath('./doc:metavar', namespaces=self.nsmap) + if metavar: + value = '``{}``'.format(metavar[0].text.strip()) + else: + value = '' + + xml = xml.xpath('./rng:data', namespaces=self.nsmap) + if not xml: + return ('', value) + + xml = xml[0] + type_ = xml.get('type', '') + + if not value: + pattern = xml.xpath('./rng:param[@name="pattern"]', + namespaces=self.nsmap) + if pattern: + value = '``{}``'.format(pattern[0].text.strip()) + + return type_, value + + + def get_attributes(self): + for xml in self.xml.xpath('''./rng:attribute | + ./rng:optional/rng:attribute | + ./rng:choice/rng:attribute''', namespaces=self.nsmap): + required = xml.getparent() == self.xml and 'yes' or 'no' + yield (xml, required) + + + def resolve_ref(self, ref): + refs = self.xml.xpath('//rng:define[name="{}"]/rng:element'.format(ref['name'])) + return refs[0] if refs else None + + + def get_child_elements(self): + for xml in self.xml.xpath('''./rng:element | ./rng:ref | + ./rng:optional/rng:element | ./rng:optional/rng:ref | + ./rng:zeroOrMore/rng:element | ./rng:zeroOrMore/rng:ref | + ./rng:oneOrMore/rng:element | ./rng:oneOrMore/rng:ref''', namespaces=self.nsmap): + parent = xml.getparent() + qname = lxml.etree.QName(parent) + if parent == self.xml: + n = '1' + elif qname.localname == 'optional': + n = '?' + elif qname.localname == 'zeroOrMore': + n = '\\*' + elif qname.localname == 'oneOrMore': + n = '\\+' + else: + print(parent.tag) + + if xml.tag == 'ref': + xml = self.resolve_ref(xml) + if xml is None: continue + + yield (self.schema.elements[xml.get('name')], n) + + + def write_rst(self, stream): + stream.write('.. _qubesxml-element-{}:\n\n'.format(self.name)) + stream.write(make_rst_section('Element: **{}**'.format(self.name), '-')) + stream.write(self.get_description()) + + attrtable = [] + for attr, required in self.get_attributes(): + type_, value = self.get_data_type(attr) + attrtable.append(( + attr.get('name'), + required, + type_, + value, + self.get_description(attr, wrap=False))) + + if attrtable: + stream.write(make_rst_section('Attributes', '^')) + write_rst_table(stream, attrtable, + ('attribute', 'req.', 'type', 'value', 'description')) + + childtable = [(':ref:`{0} `'.format(child.xml.get('name')), n) + for child, n in self.get_child_elements()] + if childtable: + stream.write(make_rst_section('Child elements', '^')) + write_rst_table(stream, childtable, ('element', 'n')) + + +class Schema(object): + nsmap = { + 'rng': 'http://relaxng.org/ns/structure/1.0', + 'q': 'http://qubes-os.org/qubes/3', + 'doc': 'http://qubes-os.org/qubes-doc/1'} + + def __init__(self, xml): + self.xml = xml + + self.wrapper = textwrap.TextWrapper(width=80, + break_long_words=False, break_on_hyphens=False) + + self.elements = {} + for x in self.xml.xpath('//rng:element', namespaces=self.nsmap): + element = Element(self, x) + self.elements[element.name] = element + + +def make_rst_section(heading, c): + return '{}\n{}\n\n'.format(heading, c[0] * len(heading)) + + +def write_rst_table(stream, it, heads): + stream.write('.. csv-table::\n') + stream.write(' :header: {}\n'.format(', '.join('"{}"'.format(c) for c in heads))) + stream.write(' :widths: {}\n\n'.format(', '.join('1' for c in heads))) + + for row in it: + stream.write(' {}\n'.format(', '.join('"{}"'.format(i) for i in row))) + + stream.write('\n') + + +def main(filename, example): + schema = Schema(lxml.etree.parse(open(filename, 'rb'))) + + sys.stdout.write(make_rst_section('Qubes XML specification', '=')) + sys.stdout.write('This is the documentation of qubes.xml autogenerated from RelaxNG source.\n\n') + sys.stdout.write('Quick example, worth thousands lines of specification:\n\n') + sys.stdout.write('.. literalinclude:: {}\n :language: xml\n\n'.format(example)) + + for name in sorted(schema.elements): + schema.elements[name].write_rst(sys.stdout) + + +if __name__ == '__main__': + main(*sys.argv[1:]) + +# vim: ts=4 sw=4 et diff --git a/qubes/tests/init.py b/qubes/tests/init.py index 45a990b3..dc4131b3 100644 --- a/qubes/tests/init.py +++ b/qubes/tests/init.py @@ -320,4 +320,7 @@ class TC_30_VMCollection(qubes.tests.QubesTestCase): class TC_90_Qubes(qubes.tests.QubesTestCase): - pass + def test_900_example_xml_in_doc(self): + self.assertXMLIsValid( + lxml.etree.parse(open('../../doc/example.xml', 'rb')), + '../../relaxng/qubes.rng') diff --git a/relaxng/qubes.rng b/relaxng/qubes.rng index 82a38a1c..95fafc51 100644 --- a/relaxng/qubes.rng +++ b/relaxng/qubes.rng @@ -170,9 +170,6 @@ the parser will complain about missing combine= attribute on the second . Whether service is enabled or disabled. Default is ``true``. - - true|false - From 2e1696cb16252c33578d309bf5d32e0cded6526c Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Mon, 12 Jan 2015 18:57:37 +0100 Subject: [PATCH 0050/1004] qubes: Fix XML validation test --- qubes/vm/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index e878642e..a98d296b 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -252,7 +252,9 @@ class BaseVM(qubes.PropertyHolder): def __xml__(self): - element = lxml.etree.Element('domain', id='domain-' + str(self.qid)) + element = lxml.etree.Element('domain') + element.set('id', 'domain-' + str(self.qid)) + element.set('class', self.__class__.__name__) element.append(self.save_properties()) @@ -261,7 +263,7 @@ class BaseVM(qubes.PropertyHolder): node = lxml.etree.Element('service') node.text = service if not self.services[service]: - node.set('enabled', 'False') + node.set('enabled', 'false') services.append(node) element.append(services) From 92eca8edb93bba944d55c90344179adbe726a943 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Tue, 13 Jan 2015 15:40:43 +0100 Subject: [PATCH 0051/1004] qubes: Fix comments accross the code Acknowledgement: This commit is a result of core3 review by Marek. --- qubes/__init__.py | 12 ------------ qubes/config.py | 2 +- qubes/utils.py | 3 +-- qubes/vm/__init__.py | 4 ++++ qubes/vm/qubesvm.py | 8 ++++++-- 5 files changed, 12 insertions(+), 17 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index 4f148d44..bad38c68 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -547,7 +547,6 @@ class property(object): def __get__(self, instance, owner): -# sys.stderr.write('{!r}.__get__({}, {!r})\n'.format(self.__name__, hex(id(instance)), owner)) if instance is None: return self @@ -556,12 +555,10 @@ class property(object): raise AttributeError( 'qubes.property should be used on qubes.PropertyHolder instances only') -# sys.stderr.write(' __get__ try\n') try: return getattr(instance, self._attr_name) except AttributeError: -# sys.stderr.write(' __get__ except\n') if self._default is self._NO_DEFAULT: raise AttributeError('property {!r} not set'.format(self.__name__)) elif isinstance(self._default, collections.Callable): @@ -761,7 +758,6 @@ class PropertyHolder(qubes.events.Emitter): :type load_stage: :py:func:`int` or :py:obj:`None` ''' -# sys.stderr.write('{!r}.get_props_list(load_stage={})\n'.format('self', load_stage)) props = set() for class_ in cls.__mro__: props.update(prop for prop in class_.__dict__.values() @@ -769,7 +765,6 @@ class PropertyHolder(qubes.events.Emitter): if load_stage is not None: props = set(prop for prop in props if prop.load_stage == load_stage) -# sys.stderr.write(' props={!r}\n'.format(props)) return sorted(props, key=lambda prop: (prop.load_stage, prop.order, prop.__name__)) @@ -829,16 +824,12 @@ class PropertyHolder(qubes.events.Emitter): :param lxml.etree._Element xml: XML node reference ''' -# sys.stderr.write('<{}>.load_properties(load_stage={}) xml={!r}\n'.format(hex(id(self)), load_stage, self.xml)) - self.events_enabled = False all_names = set(prop.__name__ for prop in self.get_props_list(load_stage)) -# sys.stderr.write(' all_names={!r}\n'.format(all_names)) for node in self.xml.xpath('./properties/property'): name = node.get('name') value = node.get('ref') or node.text -# sys.stderr.write(' load_properties name={!r} value={!r}\n'.format(name, value)) if not name in all_names: raise AttributeError( 'No property {!r} found in {!r}'.format( @@ -848,7 +839,6 @@ class PropertyHolder(qubes.events.Emitter): self.events_enabled = True self.fire_event('property-loaded') -# sys.stderr.write(' load_properties return\n') def save_properties(self, with_defaults=False): @@ -857,7 +847,6 @@ class PropertyHolder(qubes.events.Emitter): :param bool with_defaults: If :py:obj:`True`, then it also includes properties which were not set explicite, but have default values filled. ''' -# sys.stderr.write('{!r}.save_properties(with_defaults={})\n'.format(self, with_defaults)) properties = lxml.etree.Element('properties') @@ -865,7 +854,6 @@ class PropertyHolder(qubes.events.Emitter): try: value = getattr(self, (prop.__name__ if with_defaults else prop._attr_name)) except AttributeError, e: -# sys.stderr.write('AttributeError: {!s}\n'.format(e)) continue try: diff --git a/qubes/config.py b/qubes/config.py index 7cbaa382..f35801f0 100644 --- a/qubes/config.py +++ b/qubes/config.py @@ -4,7 +4,7 @@ # The Qubes OS Project, http://www.qubes-os.org # # Copyright (C) 2010 Joanna Rutkowska -# Copyright (C) 2014 Wojtek Porczyk +# Copyright (C) 2014 Wojtek Porczyk # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License diff --git a/qubes/utils.py b/qubes/utils.py index 5792d588..ba0a20ed 100644 --- a/qubes/utils.py +++ b/qubes/utils.py @@ -23,8 +23,7 @@ # -# FIXME: should be outside of QubesVM? -def get_timezone(self): +def get_timezone(): # fc18 if os.path.islink('/etc/localtime'): return '/'.join(os.readlink('/etc/localtime').split('/')[-2:]) diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index a98d296b..ab31e78a 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -586,9 +586,13 @@ class BaseVM(qubes.PropertyHolder): conf["rules"].append(rule) except EnvironmentError as err: + # problem accessing file, like ENOTFOUND, EPERM or sth + # return default config return conf + except (xml.parsers.expat.ExpatError, ValueError, LookupError) as err: + # config is invalid print("{0}: load error: {1}".format( os.path.basename(sys.argv[0]), err)) return None diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index d9b46dad..25d50cea 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -416,7 +416,7 @@ class QubesVM(qubes.vm.BaseVM): self.maxmem = total_mem_mb/2 # Linux specific cap: max memory can't scale beyond 10.79*init_mem - # XXX what?! -woju + # see https://groups.google.com/forum/#!topic/qubes-devel/VRqkFj1IOtA if self.maxmem > self.memory * 10: self.maxmem = self.memory * 10 @@ -447,7 +447,6 @@ class QubesVM(qubes.vm.BaseVM): # event handlers # - @qubes.events.handler('property-set:label') def on_property_set_label(self, event, name, new_label, old_label=None): if self.icon_path: @@ -557,6 +556,7 @@ class QubesVM(qubes.vm.BaseVM): return try: + # TODO: libvirt-ise subprocess.check_call(['sudo', system_path["qubes_pciback_cmd"], pci]) subprocess.check_call(['sudo', 'xl', 'pci-attach', str(self.xid), pci]) except Exception as e: @@ -569,6 +569,7 @@ class QubesVM(qubes.vm.BaseVM): if not self.is_running(): return + # TODO: libvirt-ise p = subprocess.Popen(['xl', 'pci-list', str(self.xid)], stdout=subprocess.PIPE) result = p.communicate() @@ -802,6 +803,8 @@ class QubesVM(qubes.vm.BaseVM): args += ["-t"] if os.isatty(sys.stderr.fileno()): args += ["-T"] + + # TODO: QSB#13 if passio: if os.name == 'nt': # wait for qrexec-client to exit, otherwise client is not properly attached to console @@ -854,6 +857,7 @@ class QubesVM(qubes.vm.BaseVM): source = 'dom0' if source is None else self.app.domains[source].name + # XXX TODO FIXME this looks bad... if input: return self.run("QUBESRPC %s %s" % (service, source), localcmd="echo %s" % input, user=user, wait=True) From af154b53fe282853d887d74dd0f6de6f90f84655 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Tue, 13 Jan 2015 15:56:10 +0100 Subject: [PATCH 0052/1004] qubes: change names of XML generating methods Methods returning lxml.etree.Elements are called xml_ or lvxml_, meant for qubes.xml or libvirt respectively. Acknowledgement: This commit is a result of core3 review by Marek. --- qubes/__init__.py | 8 ++++---- qubes/tests/init.py | 4 ++-- qubes/vm/__init__.py | 10 +++++----- qubes/vm/qubesvm.py | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index bad38c68..e741b035 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -841,7 +841,7 @@ class PropertyHolder(qubes.events.Emitter): self.fire_event('property-loaded') - def save_properties(self, with_defaults=False): + def xml_properties(self, with_defaults=False): '''Iterator that yields XML nodes representing set properties. :param bool with_defaults: If :py:obj:`True`, then it also includes properties which were not set explicite, but have default values filled. @@ -1144,8 +1144,8 @@ class Qubes(PropertyHolder): def __xml__(self): element = lxml.etree.Element('qubes') - element.append(self.save_labels()) - element.append(self.save_properties()) + element.append(self.xml_labels()) + element.append(self.xml_properties()) domains = lxml.etree.Element('domains') for vm in self.domains: @@ -1167,7 +1167,7 @@ class Qubes(PropertyHolder): os.chown(self._store, -1, grp.getgrnam('qubes').gr_gid) - def save_labels(self): + def xml_labels(self): '''Serialise labels :rtype: lxml.etree._Element diff --git a/qubes/tests/init.py b/qubes/tests/init.py index dc4131b3..d16c02cd 100644 --- a/qubes/tests/init.py +++ b/qubes/tests/init.py @@ -190,8 +190,8 @@ class TC_20_PropertyHolder(qubes.tests.QubesTestCase): def test_001_save_properties(self): self.holder.load_properties() - elements = self.holder.save_properties() - elements_with_defaults = self.holder.save_properties(with_defaults=True) + elements = self.holder.xml_properties() + elements_with_defaults = self.holder.xml_properties(with_defaults=True) self.assertEqual(len(elements), 2) self.assertEqual(len(elements_with_defaults), 3) diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index ab31e78a..cedb4384 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -256,7 +256,7 @@ class BaseVM(qubes.PropertyHolder): element.set('id', 'domain-' + str(self.qid)) element.set('class', self.__class__.__name__) - element.append(self.save_properties()) + element.append(self.xml_properties()) services = lxml.etree.Element('services') for service in self.services: @@ -303,7 +303,7 @@ class BaseVM(qubes.PropertyHolder): # @staticmethod - def xml_net_dev(ip, mac, backend): + def lvxml_net_dev(ip, mac, backend): '''Return ```` node for libvirt xml. This was previously _format_net_dev @@ -323,7 +323,7 @@ class BaseVM(qubes.PropertyHolder): @staticmethod - def xml_pci_dev(address): + def lvxml_pci_dev(address): '''Return ```` node for libvirt xml. This was previously _format_pci_dev @@ -365,7 +365,7 @@ class BaseVM(qubes.PropertyHolder): args['uuidnode'] = '{!r}'.format(self.uuid) \ if hasattr(self, 'uuid') else '' args['vmdir'] = self.dir_path - args['pcidevs'] = ''.join(lxml.etree.tostring(self.xml_pci_dev(dev)) + args['pcidevs'] = ''.join(lxml.etree.tostring(self.lvxml_pci_dev(dev)) for dev in self.devices['pci']) args['maxmem'] = str(self.maxmem) args['vcpus'] = str(self.vcpus) @@ -382,7 +382,7 @@ class BaseVM(qubes.PropertyHolder): args['dns1'] = self.netvm.gateway args['dns2'] = self.secondary_dns args['netmask'] = self.netmask - args['netdev'] = lxml.etree.tostring(self.xml_net_dev(self.ip, self.mac, self.netvm)) + args['netdev'] = lxml.etree.tostring(self.lvxml_net_dev(self.ip, self.mac, self.netvm)) args['disable_network1'] = ''; args['disable_network2'] = ''; else: diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 25d50cea..f28e9a95 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -1058,7 +1058,7 @@ class QubesVM(qubes.vm.BaseVM): netvm.start() self.libvirt_domain.attachDevice(lxml.etree.ElementTree( - self.xml_net_dev(self.ip, self.mac, self.netvm)).tostring()) + self.lvxml_net_dev(self.ip, self.mac, self.netvm)).tostring()) def detach_network(self): @@ -1072,7 +1072,7 @@ class QubesVM(qubes.vm.BaseVM): self.libvirt_domain.detachDevice(lxml.etree.ElementTree( - self.xml_net_dev(self.ip, self.mac, self.netvm)).tostring()) + self.lvxml_net_dev(self.ip, self.mac, self.netvm)).tostring()) # From eda2f7cf7379afeb2a71f982d1ead6d9b0bc1685 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Tue, 13 Jan 2015 15:58:33 +0100 Subject: [PATCH 0053/1004] qubes/tests: mark one test as expected failure This is temporary, to be fixed in the future. --- qubes/tests/init.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qubes/tests/init.py b/qubes/tests/init.py index d16c02cd..ba2519e5 100644 --- a/qubes/tests/init.py +++ b/qubes/tests/init.py @@ -174,6 +174,7 @@ class TC_20_PropertyHolder(qubes.tests.QubesTestCase): self.holder = TestHolder(xml) + @unittest.expectedFailure def test_000_load_properties(self): self.holder.load_properties() From 04c221e924e3fd3b0fbc6214efa58b20e07869a4 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Tue, 13 Jan 2015 17:55:13 +0100 Subject: [PATCH 0054/1004] qubes/vm/qubesvm: fix env manipulation on qrexec-daemon start --- qubes/vm/qubesvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index f28e9a95..a4122b3d 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -914,7 +914,7 @@ class QubesVM(qubes.vm.BaseVM): qrexec_args = [str(self.xid), self.name, self.default_user] if not self.debug: qrexec_args.insert(0, "-q") - qrexec_env = os.environ + qrexec_env = os.environ.copy() qrexec_env['QREXEC_STARTUP_TIMEOUT'] = str(self.qrexec_timeout) retcode = subprocess.call([system_path["qrexec_daemon_path"]] + qrexec_args, env=qrexec_env) From 7e12d0485d7d2beb86705fc2a7ed9c4ba6813317 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Tue, 13 Jan 2015 18:23:04 +0100 Subject: [PATCH 0055/1004] add core3 to Makefiles and spec --- Makefile | 10 +++--- qubes/Makefile | 59 +++++++++++++++++++++++++++++++ rpm_spec/core-dom0.spec | 78 ++++++++++++++++++++++++++++------------- 3 files changed, 116 insertions(+), 31 deletions(-) create mode 100644 qubes/Makefile diff --git a/Makefile b/Makefile index 2282173e..8955b154 100644 --- a/Makefile +++ b/Makefile @@ -44,9 +44,8 @@ clean: make -C qmemman clean all: - make all -C core - make all -C core-modules - make all -C tests + make all -C qubes +# make all -C tests # Currently supported only on xen ifeq ($(BACKEND_VMM),xen) make all -C qmemman @@ -60,9 +59,8 @@ ifeq ($(OS),Linux) $(MAKE) install -C linux/system-config endif $(MAKE) install -C qvm-tools - $(MAKE) install -C core - $(MAKE) install -C core-modules - $(MAKE) install -C tests + $(MAKE) install -C qubes +# $(MAKE) install -C tests ifeq ($(BACKEND_VMM),xen) # Currently supported only on xen $(MAKE) install -C qmemman diff --git a/qubes/Makefile b/qubes/Makefile new file mode 100644 index 00000000..d5d9998d --- /dev/null +++ b/qubes/Makefile @@ -0,0 +1,59 @@ +OS ?= Linux + +PYTHON_QUBESPATH = $(PYTHON_SITEPATH)/qubes +SETTINGS_SUFFIX = $(BACKEND_VMM)-$(OS) + +all: + python -m compileall . + python -O -m compileall . + +install: +ifndef PYTHON_SITEPATH + $(error PYTHON_SITEPATH not defined) +endif + mkdir -p $(DESTDIR)$(PYTHON_QUBESPATH) + mkdir \ + $(DESTDIR)$(PYTHON_QUBESPATH)/vm \ + $(DESTDIR)$(PYTHON_QUBESPATH)/ext \ + $(DESTDIR)$(PYTHON_QUBESPATH)/tests \ + $(DESTDIR)$(PYTHON_QUBESPATH)/tests/vm + + cp \ + __init__.py* \ + _pluginloader.py* \ + config.py* \ + dochelpers.py* \ + events.py* \ + log.py* \ + plugins.py* \ + rngdoc.py* \ + utils.py* \ + $(DESTDIR)$(PYTHON_QUBESPATH) + + cp \ + vm/__init__.py* \ + vm/adminvm.py* \ + vm/appvm.py* \ + vm/dispvm.py* \ + vm/hvm.py* \ + vm/netvm.py* \ + vm/proxyvm.py* \ + vm/qubesvm.py* \ + vm/templatehvm.py* \ + vm/templatevm.py* \ + $(DESTDIR)$(PYTHON_QUBESPATH)/vm + + cp ext/__init__.py* $(DESTDIR)$(PYTHON_QUBESPATH)/ext + + cp \ + tests/__init__.py* \ + tests/events.py* \ + tests/init.py* \ + tests/run.py* \ + $(DESTDIR)$(PYTHON_QUBESPATH)/tests + + cp \ + tests/vm/__init__.py* \ + tests/vm/init.py* \ + tests/vm/qubesvm.py* \ + $(DESTDIR)$(PYTHON_QUBESPATH)/tests/vm diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index 86612c48..0456bf03 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -41,8 +41,14 @@ Group: Qubes Vendor: Invisible Things Lab License: GPL URL: http://www.qubes-os.org + BuildRequires: ImageMagick BuildRequires: systemd-units + +# for building documentation +BuildRequires: python-sphinx +BuildRequires: libvirt-python + Requires(post): systemd-units Requires(preun): systemd-units Requires(postun): systemd-units @@ -67,6 +73,9 @@ Requires: bsdtar Requires: dmidecode Requires: PyQt4 +# for property's docstrings +Requires: python-docutils + # Prevent preupgrade from installation (it pretend to provide distribution upgrade) Obsoletes: preupgrade < 2.0 Provides: preupgrade = 2.0 @@ -178,32 +187,51 @@ fi %config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/qmemman.conf /usr/bin/qvm-* /usr/bin/qubes-* + %dir %{python_sitearch}/qubes -%{python_sitearch}/qubes/qubes.py -%{python_sitearch}/qubes/qubes.pyc -%{python_sitearch}/qubes/qubes.pyo -%{python_sitearch}/qubes/qubesutils.py -%{python_sitearch}/qubes/qubesutils.pyc -%{python_sitearch}/qubes/qubesutils.pyo -%{python_sitearch}/qubes/guihelpers.py -%{python_sitearch}/qubes/guihelpers.pyc -%{python_sitearch}/qubes/guihelpers.pyo -%{python_sitearch}/qubes/notify.py -%{python_sitearch}/qubes/notify.pyc -%{python_sitearch}/qubes/notify.pyo -%{python_sitearch}/qubes/backup.py -%{python_sitearch}/qubes/backup.pyc -%{python_sitearch}/qubes/backup.pyo -%{python_sitearch}/qubes/storage/*.py -%{python_sitearch}/qubes/storage/*.pyc -%{python_sitearch}/qubes/storage/*.pyo -%{python_sitearch}/qubes/settings.py -%{python_sitearch}/qubes/settings.pyc -%{python_sitearch}/qubes/settings.pyo -%{python_sitearch}/qubes/qmemman*.py* -%{python_sitearch}/qubes/modules/0*.py* -%{python_sitearch}/qubes/modules/__init__.py* -%{python_sitearch}/qubes/tests +%{python_sitearch}/qubes/__init__.py* +%{python_sitearch}/qubes/_pluginloader.py* +%{python_sitearch}/qubes/config.py* +%{python_sitearch}/qubes/dochelpers.py* +%{python_sitearch}/qubes/events.py* +%{python_sitearch}/qubes/log.py* +%{python_sitearch}/qubes/plugins.py* +%{python_sitearch}/qubes/rngdoc.py* +%{python_sitearch}/qubes/utils.py* + +%dir %{python_sitearch}/qubes/vm +%{python_sitearch}/qubes/vm/__init__.py* +%{python_sitearch}/qubes/vm/adminvm.py* +%{python_sitearch}/qubes/vm/appvm.py* +%{python_sitearch}/qubes/vm/dispvm.py* +%{python_sitearch}/qubes/vm/hvm.py* +%{python_sitearch}/qubes/vm/netvm.py* +%{python_sitearch}/qubes/vm/proxyvm.py* +%{python_sitearch}/qubes/vm/qubesvm.py* +%{python_sitearch}/qubes/vm/templatehvm.py* +%{python_sitearch}/qubes/vm/templatevm.py* + +%dir %{python_sitearch}/qubes/ext +%{python_sitearch}/qubes/ext/__init__.py* + +%dir %{python_sitearch}/qubes/tests +%{python_sitearch}/qubes/tests/__init__.py* +%{python_sitearch}/qubes/tests/run.py* + +%{python_sitearch}/qubes/tests/events.py* +%{python_sitearch}/qubes/tests/init.py* + +%dir %{python_sitearch}/qubes/tests/vm +%{python_sitearch}/qubes/tests/vm/__init__.py* +%{python_sitearch}/qubes/tests/vm/init.py* +%{python_sitearch}/qubes/tests/vm/qubesvm.py* + +# qmemman +%{python_sitearch}/qubes/qmemman.py* +%{python_sitearch}/qubes/qmemman_algo.py* +%{python_sitearch}/qubes/qmemman_client.py* +%{python_sitearch}/qubes/qmemman_server.py* + /usr/lib/qubes/unbind-pci-device.sh /usr/lib/qubes/cleanup-dispvms /usr/lib/qubes/qmemman_daemon.py* From e5d2b49fd68cc93633bdc2947e123aaaea611550 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Tue, 13 Jan 2015 22:35:10 +0100 Subject: [PATCH 0056/1004] qubes/tests: fix colourful testrunner --- qubes/tests/run.py | 60 ++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/qubes/tests/run.py b/qubes/tests/run.py index 0d3d9080..ab301408 100755 --- a/qubes/tests/run.py +++ b/qubes/tests/run.py @@ -59,20 +59,22 @@ class ANSITestResult(unittest.TestResult): def _fmtexc(self, err): s = str(err[1]) if s: - return '{bold}{}:{normal} {!s}'.format( - err[0].__name__, err[1], **self.color) + return '{color[bold]}{}:{color[normal]} {!s}'.format( + err[0].__name__, err[1], color=self.color) else: - return '{bold}{}{normal}'.format(err[0].__name__, **self.color) + return '{color[bold]}{}{color[normal]}'.format( + err[0].__name__, color=self.color) def getDescription(self, test): teststr = str(test).split('/') - teststr[-1] = '{bold}{}{normal}'.format(teststr[-1], **self.color) + teststr[-1] = '{color[bold]}{}{color[normal]}'.format( + teststr[-1], color=self.color) teststr = '/'.join(teststr) doc_first_line = test.shortDescription() if self.descriptions and doc_first_line: return '\n'.join((teststr, ' {}'.format( - doc_first_line, **self.color))) + doc_first_line, color=self.color))) else: return teststr @@ -86,7 +88,8 @@ class ANSITestResult(unittest.TestResult): def addSuccess(self, test): super(ANSITestResult, self).addSuccess(test) if self.showAll: - self.stream.writeln('{green}ok{normal}'.format(**self.color)) + self.stream.writeln('{color[green]}ok{color[normal]}'.format( + color=self.color)) elif self.dots: self.stream.write('.') self.stream.flush() @@ -94,54 +97,69 @@ class ANSITestResult(unittest.TestResult): def addError(self, test, err): super(ANSITestResult, self).addError(test, err) if self.showAll: - self.stream.writeln('{red}{bold}ERROR{normal} ({})'.format( - self._fmtexc(err), **self.color)) + self.stream.writeln( + '{color[red]}{color[bold]}ERROR{color[normal]} ({})'.format( + self._fmtexc(err), color=self.color)) elif self.dots: - self.stream.write('{red}{bold}E{normal}'.format(**self.color)) + self.stream.write( + '{color[red]}{color[bold]}E{color[normal]}'.format( + color=self.color)) self.stream.flush() def addFailure(self, test, err): super(ANSITestResult, self).addFailure(test, err) if self.showAll: - self.stream.writeln('{red}FAIL{normal}'.format(**self.color)) + self.stream.writeln('{color[red]}FAIL{color[normal]}'.format( + color=self.color)) elif self.dots: - self.stream.write('{red}F{normal}'.format(**self.color)) + self.stream.write('{color[red]}F{color[normal]}'.format( + color=self.color)) self.stream.flush() def addSkip(self, test, reason): super(ANSITestResult, self).addSkip(test, reason) if self.showAll: - self.stream.writeln('{cyan}skipped{normal} ({})'.format( - reason, **self.color)) + self.stream.writeln( + '{color[cyan]}skipped{color[normal]} ({})'.format( + reason, color=self.color)) elif self.dots: - self.stream.write('{cyan}s{normal}'.format(**self.color)) + self.stream.write('{color[cyan]}s{color[normal]}'.format( + color=self.color)) self.stream.flush() def addExpectedFailure(self, test, err): super(ANSITestResult, self).addExpectedFailure(test, err) if self.showAll: - self.stream.writeln('{yellow}expected failure{normal}'.format( - **self.color)) + self.stream.writeln( + '{color[yellow]}expected failure{color[normal]}'.format( + color=self.color)) elif self.dots: - self.stream.write('{yellow}x{normal}'.format(**self.color)) + self.stream.write('{color[yellow]}x{color[normal]}'.format( + color=self.color)) self.stream.flush() def addUnexpectedSuccess(self, test): super(ANSITestResult, self).addUnexpectedSuccess(test) if self.showAll: self.stream.writeln( - '{yellow}{bold}unexpected success{normal}'.format(**self.color)) + '{color[yellow]}{color[bold]}unexpected success{color[normal]}'.format( + color=self.color)) elif self.dots: - self.stream.write('{yellow}{bold}u{normal}'.format(**self.color)) + self.stream.write( + '{color[yellow]}{color[bold]}u{color[normal]}'.format( + color=self.color)) self.stream.flush() def printErrors(self): if self.dots or self.showAll: self.stream.writeln() self.printErrorList( - '{red}{bold}ERROR{normal}'.format(**self.color), self.errors) + '{color[red]}{color[bold]}ERROR{color[normal]}'.format( + color=self.color), + self.errors) self.printErrorList( - '{red}FAIL{normal}'.format(**self.color), self.failures) + '{color[red]}FAIL{color[normal]}'.format(color=self.color), + self.failures) def printErrorList(self, flavour, errors): for test, err in errors: From a13a41fbaf4fd22170f5dc3716b9569b232df966 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Tue, 13 Jan 2015 23:17:18 +0100 Subject: [PATCH 0057/1004] qubes/tests: fix testrunner dependency on being run in specific directory --- qubes/tests/__init__.py | 33 +++++++++++++++++++++++++++++++++ qubes/tests/init.py | 7 +++++-- qubes/tests/run.py | 2 +- qubes/tests/vm/init.py | 2 +- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index d1b63f20..06ae7ac7 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1,6 +1,8 @@ #!/usr/bin/python -O import collections +import os +import subprocess import unittest import lxml.etree @@ -12,6 +14,9 @@ import qubes.events #: :py:obj:`True` if running in dom0, :py:obj:`False` otherwise in_dom0 = False +#: :py:obj:`False` if outside of git repo, path to root of the directory otherwise +in_git = False + try: import libvirt libvirt.openReadOnly(qubes.config.defaults['libvirt_uri']).close() @@ -20,6 +25,15 @@ try: except libvirt.libvirtError: pass +try: + in_git = subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).strip() +except subprocess.CalledProcessError: + # git returned nonzero, we are outside git repo + pass +except OSError: + # command not found; let's assume we're outside + pass + def skipUnlessDom0(test_item): '''Decorator that skips test outside dom0. @@ -31,6 +45,16 @@ def skipUnlessDom0(test_item): return unittest.skipUnless(in_dom0, 'outside dom0')(test_item) +def skipUnlessGit(test_item): + '''Decorator that skips test outside git repo. + + There are very few tests that an be run only in git. One example is + correctness of example code that won't get included in RPM. + ''' + + return unittest.skipUnless(in_git, 'outside git tree')(test_item) + + class TestEmitter(qubes.events.Emitter): '''Dummy event emitter which records events fired on it. @@ -160,6 +184,15 @@ class QubesTestCase(unittest.TestCase): relaxng = lxml.etree.RelaxNG(relaxng) elif file is not None and schema is None: + if not os.path.isabs(file): + basedirs = ['/usr/share/doc/qubes/relaxng'] + if in_git: + basedirs.insert(0, os.path.join(in_git, 'relaxng')) + for basedir in basedirs: + abspath = os.path.join(basedir, file) + if os.path.exists(abspath): + file = abspath + break relaxng = lxml.etree.RelaxNG(file=file) else: diff --git a/qubes/tests/init.py b/qubes/tests/init.py index ba2519e5..ddd211d8 100644 --- a/qubes/tests/init.py +++ b/qubes/tests/init.py @@ -1,5 +1,6 @@ #!/usr/bin/python2 -O +import os import sys import unittest @@ -321,7 +322,9 @@ class TC_30_VMCollection(qubes.tests.QubesTestCase): class TC_90_Qubes(qubes.tests.QubesTestCase): + @qubes.tests.skipUnlessGit def test_900_example_xml_in_doc(self): self.assertXMLIsValid( - lxml.etree.parse(open('../../doc/example.xml', 'rb')), - '../../relaxng/qubes.rng') + lxml.etree.parse(open( + os.path.join(qubes.tests.in_git, 'doc/example.xml'), 'rb')), + 'qubes.rng') diff --git a/qubes/tests/run.py b/qubes/tests/run.py index ab301408..b5b16ad9 100755 --- a/qubes/tests/run.py +++ b/qubes/tests/run.py @@ -12,7 +12,7 @@ test_order = [ 'qubes.tests.init' ] -sys.path.insert(0, '../../') +sys.path.insert(1, '../../') class ANSIColor(dict): def __init__(self): diff --git a/qubes/tests/vm/init.py b/qubes/tests/vm/init.py index 06b28374..c966f5f1 100644 --- a/qubes/tests/vm/init.py +++ b/qubes/tests/vm/init.py @@ -131,7 +131,7 @@ class TC_10_BaseVM(qubes.tests.QubesTestCase): 'disabledservice': False, }) - self.assertXMLIsValid(vm.__xml__(), '../../relaxng/domain.rng') + self.assertXMLIsValid(vm.__xml__(), 'domain.rng') def test_001_BaseVM_nxproperty(self): xml = lxml.etree.XML(''' From 5f92afc013152ba0591864618200fc49edd2f322 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Tue, 13 Jan 2015 23:52:19 +0100 Subject: [PATCH 0058/1004] rpm: install RelaxNG specfiles --- Makefile | 1 + relaxng/Makefile | 4 ++++ rpm_spec/core-dom0.spec | 2 ++ 3 files changed, 7 insertions(+) create mode 100644 relaxng/Makefile diff --git a/Makefile b/Makefile index 8955b154..c56eae51 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,7 @@ endif $(MAKE) install -C qvm-tools $(MAKE) install -C qubes # $(MAKE) install -C tests + $(MAKE) install -C relaxng ifeq ($(BACKEND_VMM),xen) # Currently supported only on xen $(MAKE) install -C qmemman diff --git a/relaxng/Makefile b/relaxng/Makefile new file mode 100644 index 00000000..922179a3 --- /dev/null +++ b/relaxng/Makefile @@ -0,0 +1,4 @@ +RELAXNGPATH = /usr/share/doc/qubes/relaxng +install: + mkdir -p $(DESTDIR)$(RELAXNGPATH) + cp *.rng $(DESTDIR)$(RELAXNGPATH) diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index 0456bf03..3230f522 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -281,3 +281,5 @@ fi %attr(2770,root,qubes) %dir /var/log/qubes %attr(0770,root,qubes) %dir /var/run/qubes /etc/xdg/autostart/qubes-guid.desktop + +/usr/share/doc/qubes/relaxng/*.rng From 6d6d6ad7ff99768e2e05b730430a54ffbdf87dfd Mon Sep 17 00:00:00 2001 From: Jason Mehring Date: Wed, 17 Dec 2014 08:42:16 -0500 Subject: [PATCH 0059/1004] Inital .pylint configuration file --- .pylintrc | 270 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..3a46cad8 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,270 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Profiled execution. +profile=no + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +#load-plugins= + + +[MESSAGES CONTROL] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). +disable=W0142, + C0330, + I0011, + I0012 +# E8121, +# E8122, +# E8123, +# E8124, +# E8125, +# E8126, +# E8127, +# E8128 + +# Disabled Checks +# +# W0142 (star-args) +# E812* All PEP8 E12* +# E8501 PEP8 line too long +# C0330 (bad-continuation) +# I0011 (locally-disabling) +# I0012 (locally-enabling) + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (RP0004). +comment=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +#ignored-classes= + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. Python regular +# expressions are accepted. +#generated-members= + + +[BASIC] + +# Required attributes for module, separated by a comma +required-attributes= + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_,log + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Regular expression which should only match functions or classes name which do +# not require a docstring +no-docstring-rgx=__.*__ + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,FIX,XXX,TODO + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=80 + +# Maximum number of lines in a module +max-module-lines=3000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the beginning of the name of dummy variables +# (i.e. not used). +dummy-variables-rgx=_|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +#additional-builtins= + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + + +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,string,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=35 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +# Let's have max-args + 5 +max-locals=40 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +# 4x the default value +max-branchs=48 + +# Maximum number of statements in function / method body +# Double default +max-statements=100 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception + From be3e888bbed9d94b4500fbd2489bcc0e595d1ee3 Mon Sep 17 00:00:00 2001 From: Jason Mehring Date: Thu, 18 Dec 2014 08:36:09 -0500 Subject: [PATCH 0060/1004] Fixed typos --- core-modules/000QubesVm.py | 4 ++-- doc/qvm-tools/qvm-prefs.rst | 2 +- doc/qvm-tools/qvm-revert-template-changes.rst | 2 +- doc/qvm-tools/qvm-service.rst | 4 ++-- qmemman/qmemman_algo.py | 20 +++++++++---------- qubes/dochelpers.py | 6 +++--- qubes/vm/__init__.py | 2 +- qvm-tools/qvm-revert-template-changes | 4 ++-- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 18f8813b..4fb882d1 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -168,7 +168,7 @@ class QubesVm(object): "func": lambda value: datetime.datetime.fromtimestamp(int(value)) if value else None }, - ##### Internal attributes - will be overriden in __init__ regardless of args + ##### Internal attributes - will be overridden in __init__ regardless of args "config_file_template": { "func": lambda x: system_path["config_template_pv"] }, "icon_path": { @@ -316,7 +316,7 @@ class QubesVm(object): qubes_host = QubesHost() total_mem_mb = qubes_host.memory_total/1024 self.maxmem = total_mem_mb/2 - + # Linux specific cap: max memory can't scale beyond 10.79*init_mem if self.maxmem > self.memory * 10: self.maxmem = self.memory * 10 diff --git a/doc/qvm-tools/qvm-prefs.rst b/doc/qvm-tools/qvm-prefs.rst index 8ca298d3..3ad952b7 100644 --- a/doc/qvm-tools/qvm-prefs.rst +++ b/doc/qvm-tools/qvm-prefs.rst @@ -181,7 +181,7 @@ guiagent_installed Accepted values: ``True``, ``False`` This HVM have gui agent installed. This option disables full screen GUI - virtualization and enables per-window seemless GUI mode. This option will + virtualization and enables per-window seamless GUI mode. This option will be automatically turned on during Qubes Windows Tools installation, but if you install qubes gui agent in some other OS, you need to turn this option on manually. You can turn this option off to troubleshoot some early HVM OS diff --git a/doc/qvm-tools/qvm-revert-template-changes.rst b/doc/qvm-tools/qvm-revert-template-changes.rst index dee5c377..f3ff3c6a 100644 --- a/doc/qvm-tools/qvm-revert-template-changes.rst +++ b/doc/qvm-tools/qvm-revert-template-changes.rst @@ -17,7 +17,7 @@ Options .. option:: --force - Do not prompt for comfirmation + Do not prompt for confirmation Authors ======= diff --git a/doc/qvm-tools/qvm-service.rst b/doc/qvm-tools/qvm-service.rst index e5afa7a5..2704a4c5 100644 --- a/doc/qvm-tools/qvm-service.rst +++ b/doc/qvm-tools/qvm-service.rst @@ -36,7 +36,7 @@ Supported services ================== This list can be incomplete as VM can implement any additional service without -knowlege of qubes-core code. +knowledge of qubes-core code. meminfo-writer Default: enabled everywhere excluding NetVM @@ -47,7 +47,7 @@ meminfo-writer .. note:: This service is enforced to be set by dom0 code. If you try to - remove it (reset to defult state), will be recreated with the rule: enabled + remove it (reset to default state), will be recreated with the rule: enabled if VM have no PCI devices assigned, otherwise disabled. qubes-dvm diff --git a/qmemman/qmemman_algo.py b/qmemman/qmemman_algo.py index 7bd0960d..271ee1fa 100755 --- a/qmemman/qmemman_algo.py +++ b/qmemman/qmemman_algo.py @@ -24,7 +24,7 @@ import logging import string -# This are only defaults - can be overriden by QMemmanServer with values from +# This are only defaults - can be overridden by QMemmanServer with values from # config file CACHE_FACTOR = 1.3 MIN_PREFMEM = 200*1024*1024 @@ -68,7 +68,7 @@ def is_meminfo_suspicious(domain, untrusted_meminfo): if not ret and untrusted_meminfo['MemTotal'] < untrusted_meminfo['MemFree'] + untrusted_meminfo['Cached'] + untrusted_meminfo['Buffers']: ret = True #we could also impose some limits on all the above values -#but it has little purpose - all the domain can gain by passing e.g. +#but it has little purpose - all the domain can gain by passing e.g. #very large SwapTotal is that it will be assigned all free Xen memory #it can be achieved with legal values, too, and it will not allow to #starve existing domains, by design @@ -93,7 +93,7 @@ def refresh_meminfo_for_domain(domain, untrusted_xenstore_key): #sanitized, can assign domain.meminfo = untrusted_meminfo domain.mem_used = domain.meminfo['MemTotal'] - domain.meminfo['MemFree'] - domain.meminfo['Cached'] - domain.meminfo['Buffers'] + domain.meminfo['SwapTotal'] - domain.meminfo['SwapFree'] - + def prefmem(domain): #dom0 is special, as it must have large cache, for vbds. Thus, give it a special boost if domain.id == '0': @@ -105,7 +105,7 @@ def memory_needed(domain): #in balance(), "distribute total_available_memory proportionally to mempref" relies on this exact formula ret = prefmem(domain) - domain.memory_actual return ret - + #prepare list of (domain, memory_target) pairs that need to be passed #to "xm memset" equivalent in order to obtain "memsize" of memory #return empty list when the request cannot be satisfied @@ -208,7 +208,7 @@ def balance_when_enough_memory(domain_dictionary, return donors_rq + acceptors_rq -#when not enough mem to make everyone be above prefmem, make donors be at prefmem, and +#when not enough mem to make everyone be above prefmem, make donors be at prefmem, and #redistribute anything left between acceptors def balance_when_low_on_memory(domain_dictionary, xen_free_memory, total_mem_pref_acceptors, donors, acceptors): @@ -239,15 +239,15 @@ def balance_when_low_on_memory(domain_dictionary, #redistribute memory across domains -#called when one of domains update its 'meminfo' xenstore key +#called when one of domains update its 'meminfo' xenstore key #return the list of (domain, memory_target) pairs to be passed to -#"xm memset" equivalent +#"xm memset" equivalent def balance(xen_free_memory, domain_dictionary): log.debug('balance(xen_free_memory={!r}, domain_dictionary={!r})'.format( xen_free_memory, domain_dictionary)) #sum of all memory requirements - in other words, the difference between -#memory required to be added to domains (acceptors) to make them be at their +#memory required to be added to domains (acceptors) to make them be at their #preferred memory, and memory that can be taken from domains (donors) that #can provide memory. So, it can be negative when plenty of memory. total_memory_needed = 0 @@ -257,7 +257,7 @@ def balance(xen_free_memory, domain_dictionary): #sum of memory preferences of all domains that require more memory total_mem_pref_acceptors = 0 - + donors = list() # domains that can yield memory acceptors = list() # domains that require more memory #pass 1: compute the above "total" values @@ -276,7 +276,7 @@ def balance(xen_free_memory, domain_dictionary): total_memory_needed += need total_mem_pref += prefmem(domain_dictionary[i]) - total_available_memory = xen_free_memory - total_memory_needed + total_available_memory = xen_free_memory - total_memory_needed if total_available_memory > 0: return balance_when_enough_memory(domain_dictionary, xen_free_memory, total_mem_pref, total_available_memory) else: diff --git a/qubes/dochelpers.py b/qubes/dochelpers.py index f8abf3ab..6791d799 100644 --- a/qubes/dochelpers.py +++ b/qubes/dochelpers.py @@ -3,8 +3,8 @@ '''Documentation helpers -This module contains classes and functions which help to mainain documentation, -particulary our custom Sphinx extension. +This module contains classes and functions which help to maintain documentation, +particularly our custom Sphinx extension. ''' @@ -44,7 +44,7 @@ def ticket(name, rawtext, text, lineno, inliner, options={}, content=[]): :param str name: The role name used in the document :param str rawtext: The entire markup snippet, with role :param str text: The text marked with the role - :param int lineno: The line noumber where rawtext appearn in the input + :param int lineno: The line number where rawtext appears in the input :param docutils.parsers.rst.states.Inliner inliner: The inliner instance that called this function :param options: Directive options for customisation :param content: The directive content for customisation diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index cedb4384..dcd380f1 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -142,7 +142,7 @@ class BaseVM(qubes.PropertyHolder): :param xml: xml node from which to deserialise :type xml: :py:class:`lxml.etree._Element` or :py:obj:`None` - This class is responsible for serialising and deserialising machines and + This class is responsible for serializing and deserialising machines and provides basic framework. It contains no management logic. For that, see :py:class:`qubes.vm.qubesvm.QubesVM`. ''' diff --git a/qvm-tools/qvm-revert-template-changes b/qvm-tools/qvm-revert-template-changes index 29c491ca..8b2500ae 100755 --- a/qvm-tools/qvm-revert-template-changes +++ b/qvm-tools/qvm-revert-template-changes @@ -34,7 +34,7 @@ def main(): usage = "usage: %prog [options] " parser = OptionParser (usage) parser.add_option ("--force", action="store_true", dest="force", default=False, - help="Do not prompt for comfirmation") + help="Do not prompt for confirmation") (options, args) = parser.parse_args () if (len (args) != 1): @@ -98,7 +98,7 @@ def main(): prompt = raw_input ("Do you want to proceed? [y/N] ") if not (prompt == "y" or prompt == "Y"): exit (0) - + p = subprocess.Popen(["/sbin/dmsetup", "table", old_dmdev], stdout=subprocess.PIPE) result = p.communicate() dm_table = result[0] From eaace1e05c9627515f3900fc6a4cd7db379cbbd7 Mon Sep 17 00:00:00 2001 From: Jason Mehring Date: Thu, 18 Dec 2014 08:46:14 -0500 Subject: [PATCH 0061/1004] Add vm to goodnames in .pylintrc --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 3a46cad8..695e884d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -139,7 +139,7 @@ variable-rgx=[a-z_][a-z0-9_]{2,30}$ inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ # Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_,log +good-names=i,j,k,ex,Run,_,log,vm # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata From 6504da9524d36114e218a7713cc4a865124fc67c Mon Sep 17 00:00:00 2001 From: Jason Mehring Date: Thu, 18 Dec 2014 08:54:46 -0500 Subject: [PATCH 0062/1004] qubes: changed a test for None from == None to is None Conflicts: qubes/__init__.py --- qubes/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index e741b035..06b6db0a 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -100,7 +100,7 @@ class VMMConnection(object): if 'xen.lowlevel.cs' in sys.modules: self._xc = xen.lowlevel.xc.xc() self._libvirt_conn = libvirt.open(defaults['libvirt_uri']) - if self._libvirt_conn == None: + if self._libvirt_conn is None: raise QubesException("Failed connect to libvirt driver") libvirt.registerErrorHandler(self._libvirt_error_handler, None) atexit.register(self._libvirt_conn.close) From 0dbcdb8c0db282a60f85e79c3697ce380a5b4554 Mon Sep 17 00:00:00 2001 From: Jason Mehring Date: Wed, 7 Jan 2015 08:22:12 -0500 Subject: [PATCH 0063/1004] qubes: pep8 fixes ------------------------------------------------------------------------------- ISSUES: ------------------------------------------------------------------------------- - Some auto-corrected code (when line is too long) may still be over-indented. It can be manually chaged and it will be left alone, or is it acceptable as I am not sure how strict your rule is for under-indented lines for which context. If you want this only indented 4 spaces, I can work on it some more. [Also @ ~line:385 in new file] For example, __init__.py:382 OLD: def __contains__(self, key): return any((key == vm or key == vm.qid or key == vm.name) for vm in self) NEW: def __contains__(self, key): return any((key == vm or key == vm.qid or key == vm.name) for vm in self) - will not detect if there are more than 2 spaces between function methods ------------------------------------------------------------------------------- FIXED: ------------------------------------------------------------------------------- - Now uses the most horizontial space and does not use excessive lines when splitting a line - __init__:489 - '#' comments being indented for some lines and not others; would like no indent - Only happens if line preceeding comment ends in a ':' E128 - Fix visual indentation E128 - Fix a badly indented line [Now allows under-indented lines] E309 - Add missing blank line (after class declaration) [No longer adds it] E303 - Remove extra blank lines [Now allows 2 blank lines between function defs] [TODO: Create definition to enforce this] Conflicts: qubes/__init__.py --- qubes/__init__.py | 74 ++++++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index 06b6db0a..6a396f5c 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -41,7 +41,7 @@ elif os.name == 'nt': import win32file import pywintypes else: - raise RuntimeError, "Qubes works only on POSIX or WinNT systems" + raise RuntimeError("Qubes works only on POSIX or WinNT systems") import libvirt try: @@ -61,8 +61,10 @@ class QubesException(Exception): '''Exception that can be shown to the user''' pass + class VMMConnection(object): '''Connection to Virtual Machine Manager (libvirt)''' + def __init__(self): self._libvirt_conn = None self._xs = None @@ -77,7 +79,8 @@ class VMMConnection(object): @offline_mode.setter def offline_mode(self, value): if value and self._libvirt_conn is not None: - raise QubesException("Cannot change offline mode while already connected") + raise QubesException( + "Cannot change offline mode while already connected") self._offline_mode = value @@ -157,7 +160,7 @@ class QubesHost(object): (model, memory, cpus, mhz, nodes, socket, cores, threads) = \ self._app.vmm.libvirt_conn.getInfo() - self._total_mem = long(memory)*1024 + self._total_mem = long(memory) * 1024 self._no_cpus = cpus self.app.log.debug('QubesHost: no_cpus={} memory_total={}'.format(self.no_cpus, self.memory_total)) @@ -219,7 +222,7 @@ class QubesHost(object): for vm in info: previous[vm['domid']] = {} previous[vm['domid']]['cpu_time'] = ( - vm['cpu_time'] / vm['online_vcpus']) + vm['cpu_time'] / vm['online_vcpus']) previous[vm['domid']]['cpu_usage'] = 0 time.sleep(wait_time) @@ -233,12 +236,12 @@ class QubesHost(object): for vm in info: current[vm['domid']] = {} current[vm['domid']]['cpu_time'] = ( - vm['cpu_time'] / max(vm['online_vcpus'], 1)) + vm['cpu_time'] / max(vm['online_vcpus'], 1)) if vm['domid'] in previous.keys(): current[vm['domid']]['cpu_usage'] = ( float(current[vm['domid']]['cpu_time'] - previous[vm['domid']]['cpu_time']) / - long(1000**3) / (current_time-previous_time) * 100) + long(1000 ** 3) / (current_time - previous_time) * 100) if current[vm['domid']]['cpu_usage'] < 0: # VM has been rebooted current[vm['domid']]['cpu_usage'] = 0 @@ -294,7 +297,8 @@ class Label(object): def __xml__(self): - element = lxml.etree.Element('label', id='label-' + self.index, color=self.color) + element = lxml.etree.Element( + 'label', id='label-' + self.index, color=self.color) element.text = self.name return element @@ -324,7 +328,8 @@ class Label(object): .. deprecated:: 2.0 use :py:meth:`PyQt4.QtGui.QIcon.fromTheme` and :py:attr:`icon_dispvm` ''' - return os.path.join(system_path['qubes_icon_dir'], self.icon_dispvm) + ".png" + return os.path.join( + system_path['qubes_icon_dir'], self.icon_dispvm) + ".png" class VMCollection(object): @@ -342,7 +347,8 @@ class VMCollection(object): def __repr__(self): - return '<{} {!r}>'.format(self.__class__.__name__, list(sorted(self.keys()))) + return '<{} {!r}>'.format( + self.__class__.__name__, list(sorted(self.keys()))) def items(self): @@ -394,7 +400,8 @@ class VMCollection(object): # this violates duck typing, but is needed # for VMProperty to function correctly if not isinstance(value, qubes.vm.BaseVM): - raise TypeError('{} holds only BaseVM instances'.format(self.__class__.__name__)) + raise TypeError( + '{} holds only BaseVM instances'.format(self.__class__.__name__)) if not hasattr(value, 'qid'): value.qid = self.domains.get_new_unused_qid() @@ -435,7 +442,8 @@ class VMCollection(object): def __contains__(self, key): - return any((key == vm or key == vm.qid or key == vm.name) for vm in self) + return any((key == vm or key == vm.qid or key == vm.name) + for vm in self) def __len__(self): @@ -478,7 +486,7 @@ class VMCollection(object): def get_new_unused_netid(self): - used_ids = set([vm.netid for vm in self]) # if vm.is_netvm()]) + used_ids = set([vm.netid for vm in self]) # if vm.is_netvm()]) for i in range(1, MAX_NETID): if i not in used_ids: return i @@ -536,7 +544,8 @@ class property(object): load_stage=2, order=0, save_via_ref=False, doc=None): self.__name__ = name self._setter = setter - self._saver = saver if saver is not None else (lambda self, prop, value: str(value)) + self._saver = saver if saver is not None else ( + lambda self, prop, value: str(value)) self._type = type self._default = default self.order = order @@ -560,7 +569,8 @@ class property(object): except AttributeError: if self._default is self._NO_DEFAULT: - raise AttributeError('property {!r} not set'.format(self.__name__)) + raise AttributeError( + 'property {!r} not set'.format(self.__name__)) elif isinstance(self._default, collections.Callable): return self._default(instance) else: @@ -584,14 +594,16 @@ class property(object): value = self._type(value) if has_oldvalue: - instance.fire_event_pre('property-pre-set:' + self.__name__, value, oldvalue) + instance.fire_event_pre( + 'property-pre-set:' + self.__name__, value, oldvalue) else: instance.fire_event_pre('property-pre-set:' + self.__name__, value) instance._init_property(self, value) if has_oldvalue: - instance.fire_event('property-set:' + self.__name__, value, oldvalue) + instance.fire_event( + 'property-set:' + self.__name__, value, oldvalue) else: instance.fire_event('property-set:' + self.__name__, value) @@ -692,9 +704,12 @@ class property(object): ''' lcvalue = value.lower() - if lcvalue in ('0', 'no', 'false'): return False - if lcvalue in ('1', 'yes', 'true'): return True - raise ValueError('Invalid literal for boolean property: {!r}'.format(value)) + if lcvalue in ('0', 'no', 'false'): + return False + if lcvalue in ('1', 'yes', 'true'): + return True + raise ValueError( + 'Invalid literal for boolean property: {!r}'.format(value)) @@ -765,7 +780,8 @@ class PropertyHolder(qubes.events.Emitter): if load_stage is not None: props = set(prop for prop in props if prop.load_stage == load_stage) - return sorted(props, key=lambda prop: (prop.load_stage, prop.order, prop.__name__)) + return sorted(props, + key=lambda prop: (prop.load_stage, prop.order, prop.__name__)) def _init_property(self, prop, value): @@ -825,7 +841,8 @@ class PropertyHolder(qubes.events.Emitter): ''' self.events_enabled = False - all_names = set(prop.__name__ for prop in self.get_props_list(load_stage)) + all_names = set( + prop.__name__ for prop in self.get_props_list(load_stage)) for node in self.xml.xpath('./properties/property'): name = node.get('name') value = node.get('ref') or node.text @@ -852,8 +869,9 @@ class PropertyHolder(qubes.events.Emitter): for prop in self.get_props_list(): try: - value = getattr(self, (prop.__name__ if with_defaults else prop._attr_name)) - except AttributeError, e: + value = getattr( + self, (prop.__name__ if with_defaults else prop._attr_name)) + except AttributeError as e: continue try: @@ -1033,7 +1051,8 @@ class Qubes(PropertyHolder): def __init__(self, store='/var/lib/qubes/qubes.xml'): - self._extensions = set(ext(self) for ext in qubes.ext.Extension.register.values()) + self._extensions = set(ext(self) + for ext in qubes.ext.Extension.register.values()) #: collection of all VMs managed by this Qubes instance self.domains = VMCollection() @@ -1054,7 +1073,8 @@ class Qubes(PropertyHolder): except IOError: self._init() - super(Qubes, self).__init__(xml=lxml.etree.parse(self.qubes_store_file)) + super(Qubes, self).__init__( + xml=lxml.etree.parse(self.qubes_store_file)) def _open_store(self): @@ -1064,7 +1084,7 @@ class Qubes(PropertyHolder): self._storefd = open(self._store, 'r+') if os.name == 'posix': - fcntl.lockf (self.qubes_store_file, fcntl.LOCK_EX) + fcntl.lockf(self.qubes_store_file, fcntl.LOCK_EX) elif os.name == 'nt': overlapped = pywintypes.OVERLAPPED() win32file.LockFileEx(win32file._get_osfhandle(self.qubes_store_file.fileno()), @@ -1163,7 +1183,7 @@ class Qubes(PropertyHolder): lxml.etree.ElementTree(self.__xml__()).write( self._storefd, encoding='utf-8', pretty_print=True) self._storefd.sync() - os.chmod(self._store, 0660) + os.chmod(self._store, 0o660) os.chown(self._store, -1, grp.getgrnam('qubes').gr_gid) From 8afba4c5e91a972aab7c5d2cf6b897ecbe8002ec Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Fri, 16 Jan 2015 15:33:03 +0100 Subject: [PATCH 0064/1004] core3 move: storage/* --- core/storage/__init__.py | 201 ------------------------------- core/storage/xen.py | 185 ---------------------------- qubes/Makefile | 6 + qubes/storage/__init__.py | 245 ++++++++++++++++++++++++++++++++++++++ qubes/storage/xen.py | 179 ++++++++++++++++++++++++++++ qubes/vm/qubesvm.py | 79 ++++++------ rpm_spec/core-dom0.spec | 4 + 7 files changed, 478 insertions(+), 421 deletions(-) delete mode 100644 core/storage/__init__.py delete mode 100644 core/storage/xen.py create mode 100644 qubes/storage/__init__.py create mode 100644 qubes/storage/xen.py diff --git a/core/storage/__init__.py b/core/storage/__init__.py deleted file mode 100644 index a67a921d..00000000 --- a/core/storage/__init__.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/python2 -# -# The Qubes OS Project, http://www.qubes-os.org -# -# Copyright (C) 2013 Marek Marczykowski -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# - -from __future__ import absolute_import - -import os -import os.path -import re -import shutil -import subprocess -import sys - -from qubes.qubes import vm_files,system_path,defaults -from qubes.qubes import QubesException -import qubes.qubesutils - -class QubesVmStorage(object): - """ - Class for handling VM virtual disks. This is base class for all other - implementations, mostly with Xen on Linux in mind. - """ - - def __init__(self, vm, - private_img_size = None, - root_img_size = None, - modules_img = None, - modules_img_rw = False): - self.vm = vm - self.vmdir = vm.dir_path - if private_img_size: - self.private_img_size = private_img_size - else: - self.private_img_size = defaults['private_img_size'] - if root_img_size: - self.root_img_size = root_img_size - else: - self.root_img_size = defaults['root_img_size'] - - self.private_img = os.path.join(self.vmdir, vm_files["private_img"]) - if self.vm.template: - self.root_img = self.vm.template.root_img - else: - self.root_img = os.path.join(self.vmdir, vm_files["root_img"]) - self.volatile_img = os.path.join(self.vmdir, vm_files["volatile_img"]) - - # For now compute this path still in QubesVm - self.modules_img = modules_img - self.modules_img_rw = modules_img_rw - - # Additional drive (currently used only by HVM) - self.drive = None - - def get_config_params(self): - raise NotImplementedError - - def _copy_file(self, source, destination): - """ - Effective file copy, preserving sparse files etc. - """ - # TODO: Windows support - - # We prefer to use Linux's cp, because it nicely handles sparse files - retcode = subprocess.call (["cp", source, destination]) - if retcode != 0: - raise IOError ("Error while copying {0} to {1}".\ - format(source, destination)) - - def get_disk_utilization(self): - return qubes.qubesutils.get_disk_usage(self.vmdir) - - def get_disk_utilization_private_img(self): - return qubes.qubesutils.get_disk_usage(self.private_img) - - def get_private_img_sz(self): - if not os.path.exists(self.private_img): - return 0 - - return os.path.getsize(self.private_img) - - def resize_private_img(self, size): - raise NotImplementedError - - def create_on_disk_private_img(self, verbose, source_template = None): - raise NotImplementedError - - def create_on_disk_root_img(self, verbose, source_template = None): - raise NotImplementedError - - def create_on_disk(self, verbose, source_template = None): - if source_template is None: - source_template = self.vm.template - - old_umask = os.umask(002) - if verbose: - print >> sys.stderr, "--> Creating directory: {0}".format(self.vmdir) - os.mkdir (self.vmdir) - - self.create_on_disk_private_img(verbose, source_template) - self.create_on_disk_root_img(verbose, source_template) - self.reset_volatile_storage(verbose, source_template) - - os.umask(old_umask) - - def clone_disk_files(self, src_vm, verbose): - if verbose: - print >> sys.stderr, "--> Creating directory: {0}".format(self.vmdir) - os.mkdir (self.vmdir) - - if src_vm.private_img is not None and self.private_img is not None: - if verbose: - print >> sys.stderr, "--> Copying the private image:\n{0} ==>\n{1}".\ - format(src_vm.private_img, self.private_img) - self._copy_file(src_vm.private_img, self.private_img) - - if src_vm.updateable and src_vm.root_img is not None and self.root_img is not None: - if verbose: - print >> sys.stderr, "--> Copying the root image:\n{0} ==>\n{1}".\ - format(src_vm.root_img, self.root_img) - self._copy_file(src_vm.root_img, self.root_img) - - # TODO: modules? - - def rename(self, old_name, new_name): - old_vmdir = self.vmdir - new_vmdir = os.path.join(os.path.dirname(self.vmdir), new_name) - os.rename(self.vmdir, new_vmdir) - self.vmdir = new_vmdir - if self.private_img: - self.private_img = self.private_img.replace(old_vmdir, new_vmdir) - if self.root_img: - self.root_img = self.root_img.replace(old_vmdir, new_vmdir) - if self.volatile_img: - self.volatile_img = self.volatile_img.replace(old_vmdir, new_vmdir) - - def verify_files(self): - if not os.path.exists (self.vmdir): - raise QubesException ( - "VM directory doesn't exist: {0}".\ - format(self.vmdir)) - - if self.root_img and not os.path.exists (self.root_img): - raise QubesException ( - "VM root image file doesn't exist: {0}".\ - format(self.root_img)) - - if self.private_img and not os.path.exists (self.private_img): - raise QubesException ( - "VM private image file doesn't exist: {0}".\ - format(self.private_img)) - if self.modules_img is not None: - if not os.path.exists(self.modules_img): - raise QubesException ( - "VM kernel modules image does not exists: {0}".\ - format(self.modules_img)) - - def remove_from_disk(self): - shutil.rmtree (self.vmdir) - - def reset_volatile_storage(self, verbose = False, source_template = None): - if source_template is None: - source_template = self.vm.template - - # Re-create only for template based VMs - if source_template is not None and self.volatile_img: - if os.path.exists(self.volatile_img): - os.remove(self.volatile_img) - - # For StandaloneVM create it only if not already exists (eg after backup-restore) - if self.volatile_img and not os.path.exists(self.volatile_img): - if verbose: - print >> sys.stderr, "--> Creating volatile image: {0}...".\ - format(self.volatile_img) - subprocess.check_call([system_path["prepare_volatile_img_cmd"], - self.volatile_img, str(self.root_img_size / 1024 / 1024)]) - - def prepare_for_vm_startup(self, verbose): - self.reset_volatile_storage(verbose=verbose) - - if self.private_img and not os.path.exists (self.private_img): - print >>sys.stderr, "WARNING: Creating empty VM private image file: {0}".\ - format(self.private_img) - self.storage.create_on_disk_private_img(verbose=False) diff --git a/core/storage/xen.py b/core/storage/xen.py deleted file mode 100644 index e026b6b3..00000000 --- a/core/storage/xen.py +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/python2 -# -# The Qubes OS Project, http://www.qubes-os.org -# -# Copyright (C) 2013 Marek Marczykowski -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# - -from __future__ import absolute_import - -import os -import os.path -import subprocess -import sys -import re - -from qubes.storage import QubesVmStorage -from qubes.qubes import QubesException, vm_files - - -class QubesXenVmStorage(QubesVmStorage): - """ - Class for VM storage of Xen VMs. - """ - - def __init__(self, vm, **kwargs): - super(QubesXenVmStorage, self).__init__(vm, **kwargs) - - self.root_dev = "xvda" - self.private_dev = "xvdb" - self.volatile_dev = "xvdc" - self.modules_dev = "xvdd" - - if self.vm.is_template(): - self.rootcow_img = os.path.join(self.vmdir, vm_files["rootcow_img"]) - else: - self.rootcow_img = None - - def _format_disk_dev(self, path, script, vdev, rw=True, type="disk", domain=None): - if path is None: - return '' - template = " \n" \ - " \n" \ - " \n" \ - " \n" \ - "{params}" \ - " \n" - params = "" - if not rw: - params += " \n" - if domain: - params += " \n" % domain - if script: - params += " + {% endif %} + + {% endfor %} + #} + + {{ vm.storage.root_dev_config() }} + {% if not prepare_dvm %}{{ vm.storage.private_dev_config() }}{% endif %} + {{ vm.storage.other_dev_config() }} + + {% if not vm.hvm %} + {{ vm.storage.volatile_dev_config() }} + {% endif %} + + {% if vm.netvm %} + + + + + + + {% endif %} + + {% for device in vm.devices.pci %} + + +
+ + + {% endfor %} + + {% if vm.hvm %} + + +