doc: Add autogenerated qubes.xml documentation

doc/.gitignore

@@ -1 +1,3 @@
+_build
 sandbox.rst
+autoxml.rst

doc/Makefile

@@ -14,6 +14,8 @@ ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 
+DEPEND = autoxml.rst
+
 .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
 
 help:
@@ -41,40 +43,40 @@ help:
 	@echo "  install    to generate manpages and copy them to \$$(DESTDIR)/usr/share/man"
 
 clean:
-	-rm -rf $(BUILDDIR)/*
+	-rm -rf $(BUILDDIR)/* $(DEPEND)
 
-html:
+html: $(DEPEND)
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
-dirhtml:
+dirhtml: $(DEPEND)
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 
-singlehtml:
+singlehtml: $(DEPEND)
 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 	@echo
 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 
-pickle:
+pickle: $(DEPEND)
 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 	@echo
 	@echo "Build finished; now you can process the pickle files."
 
-json:
+json: $(DEPEND)
 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 	@echo
 	@echo "Build finished; now you can process the JSON files."
 
-htmlhelp:
+htmlhelp: $(DEPEND)
 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 	@echo
 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 
-qthelp:
+qthelp: $(DEPEND)
 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 	@echo
 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
@@ -83,7 +85,7 @@ qthelp:
 	@echo "To view the help file:"
 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/core-admin.qhc"
 
-devhelp:
+devhelp: $(DEPEND)
 	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
 	@echo
 	@echo "Build finished."
@@ -92,30 +94,30 @@ devhelp:
 	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/core-admin"
 	@echo "# devhelp"
 
-epub:
+epub: $(DEPEND)
 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
 	@echo
 	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
 
-latex:
+latex: $(DEPEND)
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo
 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
 	      "(use \`make latexpdf' here to do that automatically)."
 
-latexpdf:
+latexpdf: $(DEPEND)
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo "Running LaTeX files through pdflatex..."
 	$(MAKE) -C $(BUILDDIR)/latex all-pdf
 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
 
-text:
+text: $(DEPEND)
 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
 	@echo
 	@echo "Build finished. The text files are in $(BUILDDIR)/text."
 
-man:
+man: $(DEPEND)
 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
 	for file in $(BUILDDIR)/man/*.[12345678]; do \
 		gzip -f $$file; \
@@ -123,41 +125,44 @@ man:
 	@echo
 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
 
-texinfo:
+texinfo: $(DEPEND)
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
 	@echo
 	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
 	@echo "Run \`make' in that directory to run these through makeinfo" \
 	      "(use \`make info' here to do that automatically)."
 
-info:
+info: $(DEPEND)
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
 	@echo "Running Texinfo files through makeinfo..."
 	make -C $(BUILDDIR)/texinfo info
 	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
 
-gettext:
+gettext: $(DEPEND)
 	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
 	@echo
 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
 
-changes:
+changes: $(DEPEND)
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
 	@echo
 	@echo "The overview file is in $(BUILDDIR)/changes."
 
-linkcheck:
+linkcheck: $(DEPEND)
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
 	@echo
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in $(BUILDDIR)/linkcheck/output.txt."
 
-doctest:
+doctest: $(DEPEND)
 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
 	@echo "Testing of doctests in the sources finished, look at the " \
 	      "results in $(BUILDDIR)/doctest/output.txt."
 
 
+autoxml.rst: ../relaxng/qubes.rng example.xml
+	../qubes/rngdoc.py $+ > $@
+
 .PHONY: install
 install: man
 	mkdir -p $(DESTDIR)/usr/share/man/man1

doc/example.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<qubes version="3.0">
+    <properties>
+        <property name="default_netvm" ref="domain-1" />
+    </properties>
+
+    <labels>
+        <label id="label-1" color="#cc0000">red</label>
+    </labels>
+
+    <domains>
+        <domain class="QubesVM" id="domain-1">
+            <properties>
+                <property name="qid">1</property>
+                <property name="name">netvm</property>
+                <property name="label" ref="label-1" />
+            </properties>
+
+            <services>
+                <service enabled="false">meminfo-writer</service>
+                <service>qubes-firewall</service>
+            </services>
+
+            <devices class="pci">
+                <device>01:23.45</device>
+            </devices>
+        </domain>
+
+        <domain class="QubesVM" id="domain-2">
+            <properties>
+                <property name="qid">2</property>
+                <property name="name">appvm</property>
+                <property name="label" ref="label-1" />
+            </properties>
+
+            <tags>
+                <tag name="userdef">qwe123</tag>
+            </tags>
+        </domain>
+    </domains>
+</qubes>
+
+<!--
+vim: ts=4 sw=4 et
+-->

doc/index.rst

@@ -35,6 +35,11 @@ Developer documentation
    qubes-tests
    qubes-dochelpers
 
+.. toctree::
+   :maxdepth: 1
+
+   autoxml
+
 Indices and tables
 ==================
 

qubes/rngdoc.py

@@ -0,0 +1,182 @@
+#!/usr/bin/python2 -O
+
+from __future__ import print_function
+
+import sys
+import textwrap
+
+import lxml.etree
+
+class Element(object):
+    def __init__(self, schema, xml):
+        self.schema = schema
+        self.xml = xml
+
+
+    @property
+    def nsmap(self):
+        return self.schema.nsmap
+
+
+    @property
+    def name(self):
+        return self.xml.get('name')
+
+
+    def get_description(self, xml=None, wrap=True):
+        if xml is None: xml = self.xml
+
+        xml = xml.xpath('./doc:description', namespaces=self.nsmap)
+        if not xml: return ''
+        xml = xml[0]
+
+        if wrap:
+            return ''.join(self.schema.wrapper.fill(p) + '\n\n'
+                for p in textwrap.dedent(xml.text.strip('\n')).split('\n\n'))
+        else:
+            return ' '.join(xml.text.strip().split())
+
+
+    def get_data_type(self, xml=None):
+        if xml is None: xml = self.xml
+
+        value = xml.xpath('./rng:value', namespaces=self.nsmap)
+        if value:
+            value = '``{}``'.format(value[0].text.strip())
+        else:
+            metavar = xml.xpath('./doc:metavar', namespaces=self.nsmap)
+            if metavar:
+                value = '``{}``'.format(metavar[0].text.strip())
+            else:
+                value = ''
+
+        xml = xml.xpath('./rng:data', namespaces=self.nsmap)
+        if not xml:
+            return ('', value)
+
+        xml = xml[0]
+        type_ = xml.get('type', '')
+
+        if not value:
+            pattern = xml.xpath('./rng:param[@name="pattern"]',
+                namespaces=self.nsmap)
+            if pattern:
+                value = '``{}``'.format(pattern[0].text.strip())
+
+        return type_, value
+
+
+    def get_attributes(self):
+        for xml in self.xml.xpath('''./rng:attribute |
+                ./rng:optional/rng:attribute |
+                ./rng:choice/rng:attribute''', namespaces=self.nsmap):
+            required = xml.getparent() == self.xml and 'yes' or 'no'
+            yield (xml, required)
+
+
+    def resolve_ref(self, ref):
+        refs = self.xml.xpath('//rng:define[name="{}"]/rng:element'.format(ref['name']))
+        return refs[0] if refs else None
+
+
+    def get_child_elements(self):
+        for xml in self.xml.xpath('''./rng:element | ./rng:ref |
+                ./rng:optional/rng:element | ./rng:optional/rng:ref |
+                ./rng:zeroOrMore/rng:element | ./rng:zeroOrMore/rng:ref |
+                ./rng:oneOrMore/rng:element | ./rng:oneOrMore/rng:ref''', namespaces=self.nsmap):
+            parent = xml.getparent()
+            qname = lxml.etree.QName(parent)
+            if parent == self.xml:
+                n = '1'
+            elif qname.localname == 'optional':
+                n = '?'
+            elif qname.localname == 'zeroOrMore':
+                n = '\\*'
+            elif qname.localname == 'oneOrMore':
+                n = '\\+'
+            else:
+                print(parent.tag)
+
+            if xml.tag == 'ref':
+                xml = self.resolve_ref(xml)
+                if xml is None: continue
+
+            yield (self.schema.elements[xml.get('name')], n)
+
+
+    def write_rst(self, stream):
+        stream.write('.. _qubesxml-element-{}:\n\n'.format(self.name))
+        stream.write(make_rst_section('Element: **{}**'.format(self.name), '-'))
+        stream.write(self.get_description())
+
+        attrtable = []
+        for attr, required in self.get_attributes():
+            type_, value = self.get_data_type(attr)
+            attrtable.append((
+                attr.get('name'),
+                required,
+                type_,
+                value,
+                self.get_description(attr, wrap=False)))
+
+        if attrtable:
+            stream.write(make_rst_section('Attributes', '^'))
+            write_rst_table(stream, attrtable,
+                ('attribute', 'req.', 'type', 'value', 'description'))
+
+        childtable = [(':ref:`{0} <qubesxml-element-{0}>`'.format(child.xml.get('name')), n)
+            for child, n in self.get_child_elements()]
+        if childtable:
+            stream.write(make_rst_section('Child elements', '^'))
+            write_rst_table(stream, childtable, ('element', 'n'))
+
+
+class Schema(object):
+    nsmap = {
+        'rng': 'http://relaxng.org/ns/structure/1.0',
+        'q': 'http://qubes-os.org/qubes/3',
+        'doc': 'http://qubes-os.org/qubes-doc/1'}
+
+    def __init__(self, xml):
+        self.xml = xml
+
+        self.wrapper = textwrap.TextWrapper(width=80,
+            break_long_words=False, break_on_hyphens=False)
+
+        self.elements = {}
+        for x in self.xml.xpath('//rng:element', namespaces=self.nsmap):
+            element = Element(self, x)
+            self.elements[element.name] = element
+
+
+def make_rst_section(heading, c):
+    return '{}\n{}\n\n'.format(heading, c[0] * len(heading))
+
+
+def write_rst_table(stream, it, heads):
+    stream.write('.. csv-table::\n')
+    stream.write('   :header: {}\n'.format(', '.join('"{}"'.format(c) for c in heads)))
+    stream.write('   :widths: {}\n\n'.format(', '.join('1' for c in heads)))
+
+    for row in it:
+        stream.write('   {}\n'.format(', '.join('"{}"'.format(i) for i in row)))
+
+    stream.write('\n')
+
+
+def main(filename, example):
+    schema = Schema(lxml.etree.parse(open(filename, 'rb')))
+
+    sys.stdout.write(make_rst_section('Qubes XML specification', '='))
+    sys.stdout.write('This is the documentation of qubes.xml autogenerated from RelaxNG source.\n\n')
+    sys.stdout.write('Quick example, worth thousands lines of specification:\n\n')
+    sys.stdout.write('.. literalinclude:: {}\n   :language: xml\n\n'.format(example))
+
+    for name in sorted(schema.elements):
+        schema.elements[name].write_rst(sys.stdout)
+
+
+if __name__ == '__main__':
+    main(*sys.argv[1:])
+
+# vim: ts=4 sw=4 et

qubes/tests/init.py

@@ -320,4 +320,7 @@ class TC_30_VMCollection(qubes.tests.QubesTestCase):
 
 
 class TC_90_Qubes(qubes.tests.QubesTestCase):
-    pass
+    def test_900_example_xml_in_doc(self):
+        self.assertXMLIsValid(
+            lxml.etree.parse(open('../../doc/example.xml', 'rb')),
+            '../../relaxng/qubes.rng')

relaxng/qubes.rng

@@ -170,9 +170,6 @@ the parser will complain about missing combine= attribute on the second <start>.
                                         Whether service is enabled or disabled.
                                         Default is ``true``.
                                     </doc:description>
-                                    <doc:metavar>
-                                        true|false
-                                    </doc:metavar>
 
                                     <data type="boolean" />
                                 </attribute>