dochelpers.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450
  1. #
  2. # The Qubes OS Project, https://www.qubes-os.org/
  3. #
  4. # Copyright (C) 2010-2015 Joanna Rutkowska <joanna@invisiblethingslab.com>
  5. # Copyright (C) 2014-2015 Wojtek Porczyk <woju@invisiblethingslab.com>
  6. #
  7. # This program is free software; you can redistribute it and/or modify
  8. # it under the terms of the GNU General Public License as published by
  9. # the Free Software Foundation; either version 2 of the License, or
  10. # (at your option) any later version.
  11. #
  12. # This program is distributed in the hope that it will be useful,
  13. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. # GNU General Public License for more details.
  16. #
  17. # You should have received a copy of the GNU General Public License along
  18. # with this program; if not, write to the Free Software Foundation, Inc.,
  19. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  20. #
  21. '''Documentation helpers.
  22. This module contains classes and functions which help to maintain documentation,
  23. particularly our custom Sphinx extension.
  24. '''
  25. import argparse
  26. import io
  27. import json
  28. import os
  29. import re
  30. import urllib.error
  31. import urllib.request
  32. import docutils
  33. import docutils.nodes
  34. import docutils.parsers.rst
  35. import docutils.parsers.rst.roles
  36. import docutils.statemachine
  37. import sphinx
  38. import sphinx.errors
  39. import sphinx.locale
  40. import sphinx.util.docfields
  41. import qubes.tools
  42. SUBCOMMANDS_TITLE = 'COMMANDS'
  43. OPTIONS_TITLE = 'OPTIONS'
  44. class GithubTicket(object):
  45. # pylint: disable=too-few-public-methods
  46. def __init__(self, data):
  47. self.number = data['number']
  48. self.summary = data['title']
  49. self.uri = data['html_url']
  50. def fetch_ticket_info(app, number):
  51. '''Fetch info about particular trac ticket given
  52. :param app: Sphinx app object
  53. :param str number: number of the ticket, without #
  54. :rtype: mapping
  55. :raises: urllib.error.HTTPError
  56. '''
  57. response = urllib.request.urlopen(urllib.request.Request(
  58. app.config.ticket_base_uri.format(number=number),
  59. headers={
  60. 'Accept': 'application/vnd.github.v3+json',
  61. 'User-agent': __name__}))
  62. return GithubTicket(json.load(response))
  63. def ticket(name, rawtext, text, lineno, inliner, options=None, content=None):
  64. '''Link to qubes ticket
  65. :param str name: The role name used in the document
  66. :param str rawtext: The entire markup snippet, with role
  67. :param str text: The text marked with the role
  68. :param int lineno: The line number where rawtext appears in the input
  69. :param docutils.parsers.rst.states.Inliner inliner: The inliner instance \
  70. that called this function
  71. :param options: Directive options for customisation
  72. :param content: The directive content for customisation
  73. ''' # pylint: disable=unused-argument
  74. if options is None:
  75. options = {}
  76. ticketno = text.lstrip('#')
  77. if not ticketno.isdigit():
  78. msg = inliner.reporter.error(
  79. 'Invalid ticket identificator: {!r}'.format(text), line=lineno)
  80. prb = inliner.problematic(rawtext, rawtext, msg)
  81. return [prb], [msg]
  82. try:
  83. info = fetch_ticket_info(inliner.document.settings.env.app, ticketno)
  84. except urllib.error.HTTPError as e:
  85. msg = inliner.reporter.error(
  86. 'Error while fetching ticket info: {!s}'.format(e), line=lineno)
  87. prb = inliner.problematic(rawtext, rawtext, msg)
  88. return [prb], [msg]
  89. docutils.parsers.rst.roles.set_classes(options)
  90. node = docutils.nodes.reference(
  91. rawtext,
  92. '#{} ({})'.format(info.number, info.summary),
  93. refuri=info.uri,
  94. **options)
  95. return [node], []
  96. class versioncheck(docutils.nodes.warning):
  97. # pylint: disable=invalid-name
  98. pass
  99. def visit(self, node):
  100. self.visit_admonition(node, 'version')
  101. def depart(self, node):
  102. self.depart_admonition(node)
  103. sphinx.locale.admonitionlabels['version'] = 'Version mismatch'
  104. class VersionCheck(docutils.parsers.rst.Directive):
  105. '''Directive versioncheck
  106. Check if current version (from ``conf.py``) equals version specified as
  107. argument. If not, generate warning.'''
  108. has_content = True
  109. required_arguments = 1
  110. optional_arguments = 0
  111. final_argument_whitespace = True
  112. option_spec = {}
  113. def run(self):
  114. current = self.state.document.settings.env.app.config.version
  115. version = self.arguments[0]
  116. if current == version:
  117. return []
  118. text = ' '.join('''This manual page was written for version **{}**, but
  119. current version at the time when this page was generated is **{}**.
  120. This may or may not mean that page is outdated or has
  121. inconsistencies.'''.format(version, current).split())
  122. node = versioncheck(text)
  123. node['classes'] = ['admonition', 'warning']
  124. self.state.nested_parse(docutils.statemachine.StringList([text]),
  125. self.content_offset, node)
  126. return [node]
  127. def make_rst_section(heading, char):
  128. return '{}\n{}\n\n'.format(heading, char[0] * len(heading))
  129. def prepare_manpage(command):
  130. parser = qubes.tools.get_parser_for_command(command)
  131. stream = io.StringIO()
  132. stream.write('.. program:: {}\n\n'.format(command))
  133. stream.write(make_rst_section(
  134. ':program:`{}` -- {}'.format(command, parser.description), '='))
  135. stream.write('''.. warning::
  136. This page was autogenerated from command-line parser. It shouldn't be 1:1
  137. conversion, because it would add little value. Please revise it and add
  138. more descriptive help, which normally won't fit in standard ``--help``
  139. option.
  140. After rewrite, please remove this admonition.\n\n''')
  141. stream.write(make_rst_section('Synopsis', '-'))
  142. usage = ' '.join(parser.format_usage().strip().split())
  143. if usage.startswith('usage: '):
  144. usage = usage[len('usage: '):]
  145. # replace METAVARS with *METAVARS*
  146. usage = re.sub(r'\b([A-Z]{2,})\b', r'*\1*', usage)
  147. stream.write(':command:`{}` {}\n\n'.format(command, usage))
  148. stream.write(make_rst_section('Options', '-'))
  149. for action in parser._actions: # pylint: disable=protected-access
  150. stream.write('.. option:: ')
  151. if action.metavar:
  152. stream.write(', '.join('{}{}{}'.format(
  153. option,
  154. '=' if option.startswith('--') else ' ',
  155. action.metavar)
  156. for option in sorted(action.option_strings)))
  157. else:
  158. stream.write(', '.join(sorted(action.option_strings)))
  159. stream.write('\n\n {}\n\n'.format(action.help))
  160. stream.write(make_rst_section('Authors', '-'))
  161. stream.write('''\
  162. | Joanna Rutkowska <joanna at invisiblethingslab dot com>
  163. | Rafal Wojtczuk <rafal at invisiblethingslab dot com>
  164. | Marek Marczykowski <marmarek at invisiblethingslab dot com>
  165. | Wojtek Porczyk <woju at invisiblethingslab dot com>
  166. .. vim: ts=3 sw=3 et tw=80
  167. ''')
  168. return stream.getvalue()
  169. class OptionsCheckVisitor(docutils.nodes.SparseNodeVisitor):
  170. ''' Checks if the visited option nodes and the specified args are in sync.
  171. '''
  172. def __init__(self, command, args, document):
  173. assert isinstance(args, set)
  174. docutils.nodes.SparseNodeVisitor.__init__(self, document)
  175. self.command = command
  176. self.args = args
  177. def visit_desc(self, node):
  178. ''' Skips all but 'option' elements '''
  179. # pylint: disable=no-self-use
  180. if not node.get('desctype', None) == 'option':
  181. raise docutils.nodes.SkipChildren
  182. def visit_desc_name(self, node):
  183. ''' Checks if the option is defined `self.args` '''
  184. if not isinstance(node[0], docutils.nodes.Text):
  185. raise sphinx.errors.SphinxError('first child should be Text')
  186. arg = str(node[0])
  187. try:
  188. self.args.remove(arg)
  189. except KeyError:
  190. raise sphinx.errors.SphinxError(
  191. 'No such argument for {!r}: {!r}'.format(self.command, arg))
  192. def check_undocumented_arguments(self, ignored_options=None):
  193. ''' Call this to check if any undocumented arguments are left.
  194. While the documentation talks about a
  195. 'SparseNodeVisitor.depart_document()' function, this function does
  196. not exists. (For details see implementation of
  197. :py:meth:`NodeVisitor.dispatch_departure()`) So we need to
  198. manually call this.
  199. '''
  200. if ignored_options is None:
  201. ignored_options = set()
  202. left_over_args = self.args - ignored_options
  203. if left_over_args:
  204. raise sphinx.errors.SphinxError(
  205. 'Undocumented arguments for command {!r}: {!r}'.format(
  206. self.command, ', '.join(sorted(left_over_args))))
  207. class CommandCheckVisitor(docutils.nodes.SparseNodeVisitor):
  208. ''' Checks if the visited sub command section nodes and the specified sub
  209. command args are in sync.
  210. '''
  211. def __init__(self, command, sub_commands, document):
  212. docutils.nodes.SparseNodeVisitor.__init__(self, document)
  213. self.command = command
  214. self.sub_commands = sub_commands
  215. def visit_section(self, node):
  216. ''' Checks if the visited sub-command section nodes exists and it
  217. options are in sync.
  218. Uses :py:class:`OptionsCheckVisitor` for checking
  219. sub-commands options
  220. '''
  221. # pylint: disable=no-self-use
  222. title = str(node[0][0])
  223. if title.upper() == SUBCOMMANDS_TITLE:
  224. return
  225. sub_cmd = self.command + ' ' + title
  226. try:
  227. args = self.sub_commands[title]
  228. options_visitor = OptionsCheckVisitor(sub_cmd, args, self.document)
  229. node.walkabout(options_visitor)
  230. options_visitor.check_undocumented_arguments(
  231. {'--help', '--quiet', '--verbose', '-h', '-q', '-v'})
  232. del self.sub_commands[title]
  233. except KeyError:
  234. raise sphinx.errors.SphinxError(
  235. 'No such sub-command {!r}'.format(sub_cmd))
  236. def visit_Text(self, node):
  237. ''' If the visited text node starts with 'alias: ', all the provided
  238. comma separted alias in this node, are removed from
  239. `self.sub_commands`
  240. '''
  241. # pylint: disable=invalid-name
  242. text = str(node).strip()
  243. if text.startswith('aliases:'):
  244. aliases = {a.strip() for a in text.split('aliases:')[1].split(',')}
  245. for alias in aliases:
  246. assert alias in self.sub_commands
  247. del self.sub_commands[alias]
  248. def check_undocumented_sub_commands(self):
  249. ''' Call this to check if any undocumented sub_commands are left.
  250. While the documentation talks about a
  251. 'SparseNodeVisitor.depart_document()' function, this function does
  252. not exists. (For details see implementation of
  253. :py:meth:`NodeVisitor.dispatch_departure()`) So we need to
  254. manually call this.
  255. '''
  256. if self.sub_commands:
  257. raise sphinx.errors.SphinxError(
  258. 'Undocumented commands for {!r}: {!r}'.format(
  259. self.command, ', '.join(sorted(self.sub_commands.keys()))))
  260. class ManpageCheckVisitor(docutils.nodes.SparseNodeVisitor):
  261. ''' Checks if the sub-commands and options specified in the 'COMMAND' and
  262. 'OPTIONS' (case insensitve) sections in sync the command parser.
  263. '''
  264. def __init__(self, app, command, document):
  265. docutils.nodes.SparseNodeVisitor.__init__(self, document)
  266. try:
  267. parser = qubes.tools.get_parser_for_command(command)
  268. except ImportError:
  269. app.warn('cannot import module for command')
  270. self.parser = None
  271. return
  272. except AttributeError:
  273. raise sphinx.errors.SphinxError('cannot find parser in module')
  274. self.command = command
  275. self.parser = parser
  276. self.options = set()
  277. self.sub_commands = {}
  278. self.app = app
  279. # pylint: disable=protected-access
  280. for action in parser._actions:
  281. if action.help == argparse.SUPPRESS:
  282. continue
  283. if issubclass(action.__class__,
  284. qubes.tools.AliasedSubParsersAction):
  285. for cmd, cmd_parser in action._name_parser_map.items():
  286. self.sub_commands[cmd] = set()
  287. for sub_action in cmd_parser._actions:
  288. if sub_action.help != argparse.SUPPRESS:
  289. self.sub_commands[cmd].update(
  290. sub_action.option_strings)
  291. else:
  292. self.options.update(action.option_strings)
  293. def visit_section(self, node):
  294. ''' If section title is OPTIONS or COMMANDS dispatch the apropriate
  295. `NodeVisitor`.
  296. '''
  297. if self.parser is None:
  298. return
  299. section_title = str(node[0][0]).upper()
  300. if section_title == OPTIONS_TITLE:
  301. options_visitor = OptionsCheckVisitor(self.command, self.options,
  302. self.document)
  303. node.walkabout(options_visitor)
  304. options_visitor.check_undocumented_arguments()
  305. elif section_title == SUBCOMMANDS_TITLE:
  306. sub_cmd_visitor = CommandCheckVisitor(
  307. self.command, self.sub_commands, self.document)
  308. node.walkabout(sub_cmd_visitor)
  309. sub_cmd_visitor.check_undocumented_sub_commands()
  310. def check_man_args(app, doctree, docname):
  311. ''' Checks the manpage for undocumented or obsolete sub-commands and
  312. options.
  313. '''
  314. dirname, command = os.path.split(docname)
  315. if os.path.basename(dirname) != 'manpages':
  316. return
  317. app.info('Checking arguments for {!r}'.format(command))
  318. doctree.walk(ManpageCheckVisitor(app, command, doctree))
  319. #
  320. # this is lifted from sphinx' own conf.py
  321. #
  322. event_sig_re = re.compile(r'([a-zA-Z-:<>]+)\s*\((.*)\)')
  323. def parse_event(env, sig, signode):
  324. # pylint: disable=unused-argument
  325. m = event_sig_re.match(sig)
  326. if not m:
  327. signode += sphinx.addnodes.desc_name(sig, sig)
  328. return sig
  329. name, args = m.groups()
  330. signode += sphinx.addnodes.desc_name(name, name)
  331. plist = sphinx.addnodes.desc_parameterlist()
  332. for arg in args.split(','):
  333. arg = arg.strip()
  334. plist += sphinx.addnodes.desc_parameter(arg, arg)
  335. signode += plist
  336. return name
  337. #
  338. # end of codelifting
  339. #
  340. def break_to_pdb(app, *dummy):
  341. if not app.config.break_to_pdb:
  342. return
  343. import pdb
  344. pdb.set_trace()
  345. def setup(app):
  346. app.add_role('ticket', ticket)
  347. app.add_config_value(
  348. 'ticket_base_uri',
  349. 'https://api.github.com/repos/QubesOS/qubes-issues/issues/{number}',
  350. 'env')
  351. app.add_config_value('break_to_pdb', False, 'env')
  352. app.add_node(versioncheck,
  353. html=(visit, depart),
  354. man=(visit, depart))
  355. app.add_directive('versioncheck', VersionCheck)
  356. fdesc = sphinx.util.docfields.GroupedField('parameter', label='Parameters',
  357. names=['param'], can_collapse=True)
  358. app.add_object_type('event', 'event', 'pair: %s; event', parse_event,
  359. doc_field_types=[fdesc])
  360. app.connect('doctree-resolved', break_to_pdb)
  361. app.connect('doctree-resolved', check_man_args)
  362. # vim: ts=4 sw=4 et