Intersphinx features

.github/workflows/static.yaml

@@ -16,7 +16,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v2
  with:
- python-version: '3.8'
+ python-version: '3.10'
 
  - name: Install tox
  run: |

README.rst

@@ -83,6 +83,17 @@ This is the last major release to support Python 3.7.
 * `ExtRegistrar.register_post_processor()` now supports a `priority` argument that is an int.
  Highest priority callables will be called first during post-processing.
 * Fix too noisy ``--verbose`` mode (suppres some ambiguous annotations warnings).
+* Major improvements of the intersphinx integration:
+ - Pydoctor now supports linking to arbitrary intersphinx references with Sphinx role ``:external:``. 
+ - Other common Sphinx reference roles like ``:ref:``, ``:any:``, ``:class:``, ``py:*``, etc are now 
+ properly interpreted (instead of being simply stripping from the docstring).
+ - The ``--intersphinx`` option now supports the following format: ``[INVENTORY_NAME:]URL[:BASE_URL]``.
+ Where ``INVENTORY_NAME`` is a an arbitrary name used to filter ``:external:`` references, 
+ ``URL`` is an URL pointing to a ``objects.inv`` file (it can also be the base URL, ``/objects.inv`` will be added to the URL in this case).
+ It is recommended to always include the HTTP scheme in the intersphinx URLs. 
+ - The ``--intersphinx-file`` option has been added in order to load a local inventory file, this option
+ support the following format: ``[INVENTORY_NAME:]PATH:BASE_URL``. 
+ ``BASE_URL`` is the base for the generated links, it is mandatory if loading the inventory from a file.
 
 pydoctor 23.9.1
 ^^^^^^^^^^^^^^^

pydoctor/astbuilder.py

@@ -439,17 +439,10 @@ def _importNames(self, modname: str, names: Iterable[ast.alias]) -> None:
  _localNameToFullName[asname] = f'{modname}.{orgname}'
 
  def visit_Import(self, node: ast.Import) -> None:
- """Process an import statement.
-
- The grammar for the statement is roughly:
-
- mod_as := DOTTEDNAME ['as' NAME]
- import_stmt := 'import' mod_as (',' mod_as)*
+ """
+ Process an import statement.
 
- and this is translated into a node which is an instance of Import wih
- an attribute 'names', which is in turn a list of 2-tuples
- (dotted_name, as_name) where as_name is None if there was no 'as foo'
- part of the statement.
+ See L{import}. 
  """
  if not isinstance(self.builder.current, model.CanContainImportsDocumentable):
  # processing import statement in odd context

pydoctor/epydoc/markup/__init__.py

@@ -297,7 +297,11 @@ def link_to(self, target: str, label: "Flattenable") -> Tag:
  @return: The link, or just the label if the target was not found.
  """
 
- def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag:
+ def link_xref(self, target: str, label: "Flattenable", lineno: int, *, 
+ invname: Optional[str] = None,
+ domain: Optional[str] = None,
+ reftype: Optional[str] = None,
+ external: bool = False) -> Tag:
  """
  Format a cross-reference link to a Python identifier.
  This will resolve the identifier to any reasonable target,
@@ -308,6 +312,14 @@ def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag:
  @param label: The label to show for the link.
  @param lineno: The line number within the docstring at which the
  crossreference is located.
+ @param invname: In the case of an intersphinx resolution, filters by
+ inventory name. 
+ @param domain: In the case of an intersphinx resolution, filters by
+ domain. 
+ @param reftype: In the case of an intersphinx resolution, filters by
+ reference type.
+ @param external: If True, forces the lookup to use intersphinx and 
+ ingnore local names. 
  @return: The link, or just the label if the target was not found.
  In either case, the returned top-level tag will be C{<code>}.
  """

pydoctor/epydoc/markup/epytext.py

@@ -1181,20 +1181,21 @@ def _colorize_link(link: Element, token: Token, end: int, errors: List[ParseErro
 
  # Clean up the target. For URIs, assume http or mailto if they
  # don't specify (no relative urls)
- target = re.sub(r'\s', '', target)
+ # we used to stip spaces from the target here but that's no good 
+ # since intersphinx targets can contain spaces.
  if link.tag=='uri':
+ target = re.sub(r'\s', '', target)
  if not re.match(r'\w+:', target):
  if re.match(r'\w+@(\w+)(\.\w+)*', target):
  target = 'mailto:' + target
  else:
  target = 'http://'+target
  elif link.tag=='link':
- # Remove arg lists for functions (e.g., L{_colorize_link()})
- target = re.sub(r'\(.*\)$', '', target)
- if not re.match(r'^[a-zA-Z_]\w*(\.[a-zA-Z_]\w*)*$', target):
- estr = "Bad link target."
- errors.append(ColorizingError(estr, token, end))
- return
+ # Here we used to process the target in order to remove arg lists for functions
+ # and validate it. But now this happens in node2stan.parse_reference().
+ # The target is not validated anymore since the intersphinx taget names can contain any kind of text.
+ # We simply normalize it.
+ target = re.sub(r'\s', ' ', target)
 
  # Construct the target element.
  target_elt = Element('target', target, lineno=str(token.startline))

pydoctor/epydoc/markup/restructuredtext.py

@@ -39,24 +39,32 @@
 the list.
 """
 from __future__ import annotations
+from contextlib import contextmanager
+from types import ModuleType
+
 __docformat__ = 'epytext en'
 
-from typing import Iterable, List, Optional, Sequence, Set, cast
-import re
-from docutils import nodes
+from typing import Any, Iterable, Iterator, List, Optional, Sequence, Set, Tuple, cast
 
+from docutils import nodes
+from docutils.utils import SystemMessage
 from docutils.core import publish_string
 from docutils.writers import Writer
-from docutils.parsers.rst.directives.admonitions import BaseAdmonition # type: ignore[import]
+from docutils.parsers.rst.directives.admonitions import BaseAdmonition # type: ignore[import-untyped]
 from docutils.readers.standalone import Reader as StandaloneReader
 from docutils.utils import Reporter
 from docutils.parsers.rst import Directive, directives
 from docutils.transforms import Transform, frontmatter
+from docutils.parsers.rst import roles
+import docutils.parsers.rst.states
 
 from pydoctor.epydoc.markup import Field, ParseError, ParsedDocstring, ParserFunction
 from pydoctor.epydoc.markup.plaintext import ParsedPlaintextDocstring
-from pydoctor.epydoc.docutils import new_document
+from pydoctor.epydoc.docutils import new_document, set_node_attributes
 from pydoctor.model import Documentable
+from pydoctor.sphinx import (ALL_SUPPORTED_ROLES, SUPPORTED_DEFAULT_REFTYPES, 
+ SUPPORTED_DOMAINS, SUPPORTED_EXTERNAL_DOMAINS, 
+ SUPPORTED_EXTERNAL_STD_REFTYPES, parse_domain_reftype)
 
 #: A dictionary whose keys are the "consolidated fields" that are
 #: recognized by epydoc; and whose values are the corresponding epydoc
@@ -93,18 +101,11 @@
  """
  writer = _DocumentPseudoWriter()
  reader = _EpydocReader(errors) # Outputs errors to the list.
-
- # Credits: mhils - Maximilian Hils from the pdoc repository https://github.com/mitmproxy/pdoc
- # Strip Sphinx interpreted text roles for code references: :obj:`foo` -> `foo`
- docstring = re.sub(
- r"(:py)?:(mod|func|data|const|class|meth|attr|exc|obj):", "", docstring
- )
-
- publish_string(docstring, writer=writer, reader=reader,
- settings_overrides={'report_level':10000,
- 'halt_level':10000,
- 'warning_stream':None})
-
+ with patch_docutils_role_function(errors):
+ publish_string(docstring, writer=writer, reader=reader,
+ settings_overrides={'report_level':10000,
+ 'halt_level':10000,
+ 'warning_stream':None})
  document = writer.document
  visitor = _SplitFieldsTranslator(document, errors)
  document.walk(visitor)
@@ -498,6 +499,131 @@
  'caption': directives.unchanged_required,
  }
 
+def parse_external(name: str) -> Tuple[Optional[str], Optional[str]]:
+ """
+ Returns a tuple: (inventory name, role)
+
+ @raises ValueError: If the format is invalid.
+ """
+ assert name.startswith('external'), name
+ # either we have an explicit inventory name, i.e,
+ # :external+inv:reftype: or
+ # :external+inv:domain:reftype:
+ # or we look in all inventories, i.e.,
+ # :external:reftype: or
+ # :external:domain:reftype: or
+ # :external: 
+ suffix = name[9:]
+ if len(name) > len('external'):
+ if name[8] == '+':
+ parts = suffix.split(':', 1)
+ if len(parts) == 2:
+ inv_name, suffix = parts
+ if inv_name and suffix:
+ return inv_name, suffix
+ elif len(parts) == 1:
+ inv_name, = parts
+ if inv_name:
+ return inv_name, None
+ elif name[8] == ':' and suffix:
+ return None, suffix
+ msg = f'Malformed :external: role name: {name!r}'
+ raise ValueError(msg)
+ return None, None
+
+class LinkRole:
+ def __init__(self, errors: List[ParseError]) -> None:
+ self.errors = errors
+
+ # roles._RoleFn
+ def __call__(self, role: str, rawtext: str, text: str, lineno: int, 
+ inliner: docutils.parsers.rst.states.Inliner,
+ options:Any=None, content:Any=None) -> 'tuple[list[nodes.Node], list[nodes.Node]]':
+
+ # See https://www.sphinx-doc.org/en/master/usage/referencing.html
+ # and https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html
+ invname: Optional[str] = None
+ domain: Optional[str] = None
+ reftype: Optional[str] = None
+ external: bool = False
+ if role.startswith('external'):
+ try:
+ invname, suffix = parse_external(role)
+ if suffix is not None:
+ domain, reftype = parse_domain_reftype(suffix)
+ except ValueError as e:
+ self.errors.append(ParseError(str(e), lineno, is_fatal=False))
+ return [], []
+ else:
+ external = True
+ elif role:
+ try:
+ domain, reftype = parse_domain_reftype(role)
+ except ValueError as e:
+ self.errors.append(ParseError(str(e), lineno, is_fatal=False))
+ return [], []
+
+ if reftype in SUPPORTED_DOMAINS and domain is None:
+ self.errors.append(ParseError('Malformed role name, domain is missing reference type', 
+ lineno, is_fatal=False))
+ return [], []
+
+ if reftype in SUPPORTED_DEFAULT_REFTYPES:
+ reftype = None
+
+ if reftype in SUPPORTED_EXTERNAL_STD_REFTYPES and domain is None:
+ external = True
+ domain = 'std'
+
+ if domain in SUPPORTED_EXTERNAL_DOMAINS:
+ external = True
+
+ text_node = nodes.Text(text)
+ node = nodes.title_reference(rawtext, '', 
+ invname=invname,
+ domain=domain,
+ reftype=reftype,
+ external=external,
+ lineno=lineno)
+
+ set_node_attributes(node, children=[text_node], document=inliner.document) # type: ignore
+ return [node], []
+
+@contextmanager
+def patch_docutils_role_function(errors:List[ParseError]) -> Iterator[None]:
+ r"""
+ Like sphinx, we are patching the L{docutils.parsers.rst.roles.role} function. 
+ This function is a factory for role handlers functions. In order to handle any kind
+ of roles names like C{:external+python:doc:`something`} (the role here is C{external+python:doc}, 
+ we need to patch this function because Docutils only handles extact matches...
+
+ Tip: To list roles contained in a given inventory, use the following command::
+
+ python3 -m sphinx.ext.intersphinx https://docs.python.org/3/objects.inv | grep -v '^\s'
+
+ """
+
+ old_role = roles.role
+
+ def new_role(role_name: str, language_module: ModuleType, 
+ lineno: int, reporter: Reporter) -> 'tuple[nodes._RoleFn, list[SystemMessage]]':
+
+ if role_name in ALL_SUPPORTED_ROLES or any(
+ role_name.startswith(f'{n}:') for n in ALL_SUPPORTED_ROLES) or \
+ role_name.startswith('external+'): # 'external+' is a special case
+ return LinkRole(errors), []
+
+ return old_role(role_name, language_module, lineno, reporter) # type: ignore
+
+ roles.role = new_role
+ yield
+ roles.role = old_role
+
+# https://docutils.sourceforge.io/docs/ref/rst/directives.html#default-role
+# there is no possible code path that triggers messages from the default role, 
+# so that's ok to use an anonymous list here
+roles.register_local_role('default-role', LinkRole([]))
+
 directives.register_directive('python', PythonCodeDirective)
 directives.register_directive('code', DocutilsAndSphinxCodeBlockAdapter)
 directives.register_directive('code-block', DocutilsAndSphinxCodeBlockAdapter)