From 8732a70f235872afd73da6072a100dece6d19655 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Mon, 8 Feb 2021 17:55:33 +0000 Subject: [PATCH 01/41] Add domain support for directives (#101) - This starts working towards #71, by making directive completions domain aware - The `std` and "default" domain directives are suggested without the namespace prefix - The `primary_domain` config option is respected. Also - Start adding implementation notes to the docs - Refactored code again so that rather than structuring code according to lsp features (completions etc.), it's structured according to rst language features (directives, roles etc) - Cache sphinx app instances in the `sphinx` fixture, with `intersphinx` hitting the network on initialization it makes a big difference to test duration --- docs/conf.py | 23 +- docs/contributing/lsp.rst | 11 + docs/contributing/lsp/directives.rst | 5 + docs/contributing/lsp/testing.rst | 4 + lib/esbonio/changes/101.feature.rst | 1 + lib/esbonio/esbonio/lsp/__init__.py | 23 +- .../esbonio/lsp/completion/__init__.py | 1 - .../esbonio/lsp/completion/directives.py | 154 --------- lib/esbonio/esbonio/lsp/directives.py | 298 ++++++++++++++++++ .../esbonio/lsp/{completion => }/roles.py | 7 +- lib/esbonio/esbonio/lsp/testing.py | 83 +++++ lib/esbonio/tests/conftest.py | 11 +- .../tests/data/sphinx-extensions/conf.py | 3 + .../sphinx-extensions/theorems/pythagoras.rst | 24 +- .../tests/lsp/completion/test_directives.py | 46 --- .../tests/lsp/completion/test_integration.py | 74 +---- lib/esbonio/tests/lsp/test_directives.py | 185 +++++++++++ .../tests/lsp/{completion => }/test_roles.py | 2 +- 18 files changed, 655 insertions(+), 300 deletions(-) create mode 100644 docs/contributing/lsp.rst create mode 100644 docs/contributing/lsp/directives.rst create mode 100644 docs/contributing/lsp/testing.rst create mode 100644 lib/esbonio/changes/101.feature.rst delete mode 100644 lib/esbonio/esbonio/lsp/completion/__init__.py delete mode 100644 lib/esbonio/esbonio/lsp/completion/directives.py create mode 100644 lib/esbonio/esbonio/lsp/directives.py rename lib/esbonio/esbonio/lsp/{completion => }/roles.py (98%) create mode 100644 lib/esbonio/esbonio/lsp/testing.py delete mode 100644 lib/esbonio/tests/lsp/completion/test_directives.py create mode 100644 lib/esbonio/tests/lsp/test_directives.py rename lib/esbonio/tests/lsp/{completion => }/test_roles.py (98%) diff --git a/docs/conf.py b/docs/conf.py index a847d81e..b06f4242 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,20 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.intersphinx", "esbonio.tutorial"] +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "esbonio.tutorial", +] + +autodoc_default_options = { + "members": True, + "member-order": "bysource", + "undoc-members": True, +} +autodoc_typehints = "description" intersphinx_mapping = { "python": ("https://docs.python.org/3/", None), @@ -52,7 +65,13 @@ # a list of builtin themes. # html_theme = "sphinx_rtd_theme" - +html_context = { + "conf_py_path": "/docs/", + "display_github": True, + "github_repo": "esbonio", + "github_user": "swyddfa", + "github_version": "release", +} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". diff --git a/docs/contributing/lsp.rst b/docs/contributing/lsp.rst new file mode 100644 index 00000000..170f4b49 --- /dev/null +++ b/docs/contributing/lsp.rst @@ -0,0 +1,11 @@ +Language Server +=============== + +This section contains autogenerated implementation notes based on docstrings in +the codebase. + +.. toctree:: + :maxdepth: 2 + :glob: + + lsp/* \ No newline at end of file diff --git a/docs/contributing/lsp/directives.rst b/docs/contributing/lsp/directives.rst new file mode 100644 index 00000000..8711b72a --- /dev/null +++ b/docs/contributing/lsp/directives.rst @@ -0,0 +1,5 @@ +Directives +========== + +.. automodule:: esbonio.lsp.directives + :members: \ No newline at end of file diff --git a/docs/contributing/lsp/testing.rst b/docs/contributing/lsp/testing.rst new file mode 100644 index 00000000..aece4d52 --- /dev/null +++ b/docs/contributing/lsp/testing.rst @@ -0,0 +1,4 @@ +Testing +======= + +.. automodule:: esbonio.lsp.testing diff --git a/lib/esbonio/changes/101.feature.rst b/lib/esbonio/changes/101.feature.rst new file mode 100644 index 00000000..d1e930b7 --- /dev/null +++ b/lib/esbonio/changes/101.feature.rst @@ -0,0 +1 @@ +Directive completions are now domain aware. \ No newline at end of file diff --git a/lib/esbonio/esbonio/lsp/__init__.py b/lib/esbonio/esbonio/lsp/__init__.py index f42b092a..b2d3fe1f 100644 --- a/lib/esbonio/esbonio/lsp/__init__.py +++ b/lib/esbonio/esbonio/lsp/__init__.py @@ -1,7 +1,9 @@ +# from __future__ import annotations + import importlib import logging -from typing import List +from typing import List, Optional from pygls.features import COMPLETION, INITIALIZE, INITIALIZED, TEXT_DOCUMENT_DID_SAVE from pygls.server import LanguageServer @@ -13,15 +15,24 @@ Position, ) from pygls.workspace import Document +from sphinx.application import Sphinx BUILTIN_MODULES = [ "esbonio.lsp.sphinx", - "esbonio.lsp.completion.directives", - "esbonio.lsp.completion.roles", + "esbonio.lsp.directives", + "esbonio.lsp.roles", ] +class LanguageFeature: + """Base class for language features.""" + + def __init__(self, rst: "RstLanguageServer"): + self.rst = rst + self.logger = rst.logger.getChild(self.__class__.__name__) + + class RstLanguageServer(LanguageServer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -29,7 +40,7 @@ def __init__(self, *args, **kwargs): self.logger = logging.getLogger(__name__) """The logger that should be used for all Language Server log entries""" - self.app = None + self.app: Optional[Sphinx] = None """Sphinx application instance configured for the current project.""" self.on_init_hooks = [] @@ -96,9 +107,7 @@ def create_language_server(modules: List[str]) -> RstLanguageServer: modules: The list of modules that should be loaded. """ - import asyncio - - server = RstLanguageServer(asyncio.new_event_loop()) + server = RstLanguageServer() for mod in modules: server.load_module(mod) diff --git a/lib/esbonio/esbonio/lsp/completion/__init__.py b/lib/esbonio/esbonio/lsp/completion/__init__.py deleted file mode 100644 index 369e399f..00000000 --- a/lib/esbonio/esbonio/lsp/completion/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Completions.""" diff --git a/lib/esbonio/esbonio/lsp/completion/directives.py b/lib/esbonio/esbonio/lsp/completion/directives.py deleted file mode 100644 index 6fe069ce..00000000 --- a/lib/esbonio/esbonio/lsp/completion/directives.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Logic around directive completions goes here.""" -import importlib -import inspect -import re - -from typing import List - -from docutils.parsers.rst import directives -from pygls.types import CompletionItem, CompletionItemKind, InsertTextFormat - -from esbonio.lsp import RstLanguageServer - - -def resolve_directive(directive): - - # 'Core' docutils directives are returned as tuples (modulename, ClassName) - # so its up to us to resolve the reference - if isinstance(directive, tuple): - mod, cls = directive - - modulename = "docutils.parsers.rst.directives.{}".format(mod) - module = importlib.import_module(modulename) - directive = getattr(module, cls) - - return directive - - -def directive_to_completion_item(name: str, directive) -> CompletionItem: - """Convert an rst directive to its CompletionItem representation.""" - - directive = resolve_directive(directive) - documentation = inspect.getdoc(directive) - - # TODO: Give better names to arguments based on what they represent. - args = " ".join( - "${{{0}:arg{0}}}".format(i) for i in range(1, directive.required_arguments + 1) - ) - snippet = " {}:: {}$0".format(name, args) - - return CompletionItem( - name, - kind=CompletionItemKind.Class, - detail="directive", - documentation=documentation, - insert_text=snippet, - insert_text_format=InsertTextFormat.Snippet, - ) - - -def options_to_completion_items(directive) -> List[CompletionItem]: - """Convert a directive's options to a list of completion items.""" - - directive = resolve_directive(directive) - options = directive.option_spec - - if options is None: - return [] - - return [ - CompletionItem( - opt, detail="option", kind=CompletionItemKind.Field, insert_text=f"{opt}:" - ) - for opt in options - ] - - -class DirectiveCompletion: - """A completion handler for directives.""" - - def __init__(self, rst: RstLanguageServer): - self.rst = rst - - def initialize(self): - self.discover() - - def discover(self): - std_directives = {} - py_directives = {} - - # Find directives that have been registered directly with docutils. - dirs = {**directives._directive_registry, **directives._directives} - - if self.rst.app is not None: - - # Find directives that are held in a Sphinx domain. - # TODO: Implement proper domain handling, will focus on std + python for now - domains = self.rst.app.registry.domains - std_directives = domains["std"].directives - py_directives = domains["py"].directives - - dirs = {**dirs, **std_directives, **py_directives} - - self.directives = { - k: directive_to_completion_item(k, v) - for k, v in dirs.items() - if k != "restructuredtext-test-directive" - } - - self.options = { - k: options_to_completion_items(v) - for k, v in dirs.items() - if k in self.directives - } - - self.rst.logger.debug("Discovered %s directives", len(self.directives)) - - suggest_triggers = [ - re.compile( - r""" - ^\s* # directives may be indented - \.\. # they start with an rst comment - [ ]* # followed by a space - (?P[\w-]+)?$ # with an optional name - """, - re.VERBOSE, - ), - re.compile( - r""" - (?P\s+) # directive options must only be preceeded by whitespace - : # they start with a ':' - (?P[\w-]*) # they have a name - $ - """, - re.VERBOSE, - ), - ] - - def suggest(self, match, doc, position) -> List[CompletionItem]: - groups = match.groupdict() - - if "indent" not in groups: - return list(self.directives.values()) - - # Search backwards so that we can determine the context for our completion - indent = groups["indent"] - linum = position.line - 1 - line = doc.lines[linum] - - while line.startswith(indent): - linum -= 1 - line = doc.lines[linum] - - # Only offer completions if we're within a directive's option block - match = re.match(r"\s*\.\.[ ]*(?P[\w-]+)::", line) - if not match: - return [] - - return self.options.get(match.group("name"), []) - - -def setup(rst: RstLanguageServer): - - directive_completion = DirectiveCompletion(rst) - rst.add_feature(directive_completion) diff --git a/lib/esbonio/esbonio/lsp/directives.py b/lib/esbonio/esbonio/lsp/directives.py new file mode 100644 index 00000000..e28121e5 --- /dev/null +++ b/lib/esbonio/esbonio/lsp/directives.py @@ -0,0 +1,298 @@ +"""Logic around directive completions goes here.""" +import importlib +import inspect +import re + +from typing import List, Union, Tuple + +from docutils.parsers.rst import directives, Directive +from pygls.types import ( + CompletionItem, + CompletionItemKind, + InsertTextFormat, + Position, + Range, + TextEdit, +) +from pygls.workspace import Document + +from esbonio.lsp import RstLanguageServer, LanguageFeature + +DIRECTIVE = re.compile(r"\s*\.\.[ ](?P[\w]+:)?(?P[\w-]+)::") +"""A regular expression that matches a complete, valid directive declaration. Not +including the arguments or options.""" + +PARTIAL_DIRECTIVE = re.compile( + r""" + (?P^\s*) # directives can be indented + \.\. # start with a commment + (?P[ ]?) # may have a space + (?P[\w]+:)? # with an optional domain namespace + (?P[\w-]+)? # with an optional name + $ + """, + re.VERBOSE, +) +"""A regular expression that matches a partial directive declaraiton. Used when +generating auto complete suggestions.""" + +PARTIAL_DIRECTIVE_OPTION = re.compile( + r""" + (?P\s+) # directive options must only be preceeded by whitespace + : # they start with a ':' + (?P[\w-]*) # they have a name + $ + """, + re.VERBOSE, +) +"""A regular expression that matches a partial directive option. Used when generating +auto complete suggestions.""" + + +class Directives(LanguageFeature): + """Directive support for the language server.""" + + def initialize(self): + self.discover() + + def discover(self): + """Build an index of all available directives and the their options.""" + ignored_directives = ["restructuredtext-test-directive"] + + # Find directives that have been registered directly with docutils. + dirs = {**directives._directive_registry, **directives._directives} + + # Find directives under Sphinx domains + if self.rst.app is not None: + + domains = self.rst.app.registry.domains + primary_domain = self.rst.app.config.primary_domain + + for name, domain in domains.items(): + namefmt = "{name}:{dirname}" + + # The "standard" domain and the "primary_domain" do not require + # the prefix + if name == "std" or name == primary_domain: + namefmt = "{dirname}" + + dirs.update( + { + namefmt.format(name=name, dirname=dirname): directive + for dirname, directive in domain.directives.items() + } + ) + + self.directives = { + k: self.resolve_directive(v) + for k, v in dirs.items() + if k not in ignored_directives + } + + self.options = { + k: self.options_to_completion_items(v) for k, v in self.directives.items() + } + + self.logger.info("Discovered %s directives", len(self.directives)) + self.logger.debug(self.directives.keys()) + + def resolve_directive(self, directive: Union[Directive, Tuple[str]]): + + # 'Core' docutils directives are returned as tuples (modulename, ClassName) + # so its up to us to resolve the reference + if isinstance(directive, tuple): + mod, cls = directive + + modulename = "docutils.parsers.rst.directives.{}".format(mod) + module = importlib.import_module(modulename) + directive = getattr(module, cls) + + return directive + + suggest_triggers = [PARTIAL_DIRECTIVE, PARTIAL_DIRECTIVE_OPTION] + """Regular expressions that match lines that we want to offer autocomplete + suggestions for.""" + + def suggest( + self, match: "re.Match", doc: Document, position: Position + ) -> List[CompletionItem]: + self.logger.debug("Trigger match: %s", match) + groups = match.groupdict() + + if "domain" in groups: + return self.suggest_directives(match, position) + + return self.suggest_options(match, doc, position) + + def suggest_directives(self, match, position) -> List[CompletionItem]: + self.logger.debug("Suggesting directives") + + domain = match.groupdict()["domain"] or "" + items = [] + + for name, directive in self.directives.items(): + + if not name.startswith(domain): + continue + + item = self.directive_to_completion_item(name, directive, match, position) + items.append(item) + + return items + + def suggest_options( + self, match: "re.Match", doc: Document, position: Position + ) -> List[CompletionItem]: + + groups = match.groupdict() + + self.logger.debug("Suggesting options") + self.logger.debug("Match groups: %s", groups) + + indent = groups["indent"] + + # Search backwards so that we can determine the context for our completion + linum = position.line - 1 + line = doc.lines[linum] + + while line.startswith(indent): + linum -= 1 + line = doc.lines[linum] + + # Only offer completions if we're within a directive's option block + match = DIRECTIVE.match(line) + + self.logger.debug("Context line: %s", line) + self.logger.debug("Context match: %s", match) + + if not match: + return [] + + domain = match.group("domain") or "" + name = f"{domain}{match.group('name')}" + + self.logger.debug("Returning options for directive: %s", name) + return self.options.get(name, []) + + def directive_to_completion_item( + self, name: str, directive: Directive, match: "re.Match", position: Position + ) -> CompletionItem: + """Convert an rst directive to its CompletionItem representation. + + Previously, it was fine to pre-convert directives into their completion item + representation during the :meth:`discover` phase. However a number of factors + combined to force this to be something we have to compute specifically for each + completion site. + + It all stems from directives that live under a namespaced domain e.g. + ``.. c:macro::``. First in order to get trigger character completions for + directives, we need to allow users to start typing the directive name + immediately after the second dot and have the CompletionItem insert the leading + space. Which is exactly what we used to do, setting + ``insert_text=" directive::"`` and we were done. + + However with domain support, we introduced the possibility of a ``:`` character + in the name of a directive. You can imagine a scenario where a user types in a + domain namespace, say ``py:`` in order to filter down the list of options to + directives that belong to that namespace. With ``:`` being a trigger character + for role completions and the like, this would cause editors like VSCode to issue + a new completion request ignoring the old one. + + That isn't necessarily the end of the world, but with CompletionItems assuming + that they were following the ``..`` characters, the ``insert_text`` was no + longer correct leading to broken completions like ``..py: py:function::``. + + In order to handle the two scenarios, conceptually the easiest approach is to + switch to using a ``text_edit`` and replace the entire line with the correct + text. Unfortunately in practice this was rather fiddly. + + Upon first setting the ``text_edit`` field VSCode suddenly stopped presenting + any options! After much debugging, head scratching and searching, I eventually + found a `couple `_ of + `issues `_ that hinted as to + what was happening. + + I **think** what happens is that since the ``range`` of the text edit extends + back to the start of the line VSCode considers the entire line to be the filter + for the CompletionItems so it's looking to select items that start with ``..`` + - which is none of them! + + To work around this, we additionaly need to set the ``filter_text`` field so + that VSCode computes matches against that instead of the label. Then in order + for the items to be shown the value of that field needs to be ``..my:directive`` + so it corresponds with what the user has actually written. + + Parameters + ---------- + name: + The name of the directive as a user would type in an rst document + directive: + The class definition that implements the Directive's behavior + match: + The regular expression match object that represents the line we are providing + the autocomplete suggestions for. + position: + The position in the source code where the autocompletion request was sent + from. + """ + groups = match.groupdict() + prefix = groups["prefix"] + indent = groups["indent"] + + documentation = inspect.getdoc(directive) + + # Ignore directives that do not provide their own documentation. + if documentation.startswith("Base class for reStructedText directives."): + documentation = None + + # TODO: Give better names to arguments based on what they represent. + args = " ".join( + "${{{0}:arg{0}}}".format(i) + for i in range(1, directive.required_arguments + 1) + ) + + return CompletionItem( + name, + kind=CompletionItemKind.Class, + detail="directive", + documentation=documentation, + filter_text=f"..{prefix}{name}", + insert_text_format=InsertTextFormat.Snippet, + text_edit=TextEdit( + range=Range( + Position(position.line, 0), + Position(position.line, position.character - 1), + ), + new_text=f"{indent}.. {name}:: {args}", + ), + ) + + def options_to_completion_items(self, directive: Directive) -> List[CompletionItem]: + """Convert a directive's options to a list of completion items. + + Parameters + ---------- + directive: + The directive whose options we are creating completions for. + """ + + options = directive.option_spec + + if options is None: + return [] + + return [ + CompletionItem( + opt, + detail="option", + kind=CompletionItemKind.Field, + insert_text=f"{opt}: ", + ) + for opt in options + ] + + +def setup(rst: RstLanguageServer): + + directive_completion = Directives(rst) + rst.add_feature(directive_completion) diff --git a/lib/esbonio/esbonio/lsp/completion/roles.py b/lib/esbonio/esbonio/lsp/roles.py similarity index 98% rename from lib/esbonio/esbonio/lsp/completion/roles.py rename to lib/esbonio/esbonio/lsp/roles.py index cc730077..98f710db 100644 --- a/lib/esbonio/esbonio/lsp/completion/roles.py +++ b/lib/esbonio/esbonio/lsp/roles.py @@ -8,13 +8,12 @@ from sphinx.domains import Domain from esbonio.lsp import RstLanguageServer +from esbonio.lsp.directives import DIRECTIVE def namespace_to_completion_item(namespace: str) -> CompletionItem: return CompletionItem( - namespace, - detail="intersphinx namespace", - kind=CompletionItemKind.Module, + namespace, detail="intersphinx namespace", kind=CompletionItemKind.Module, ) @@ -134,7 +133,7 @@ def suggest(self, match, doc, position) -> List[CompletionItem]: # Unless we are within a directive's options block, we should offer role # suggestions - if re.match(r"\s*\.\.[ ]*([\w-]+)::", line): + if DIRECTIVE.match(line): return [] return list(self.roles.values()) diff --git a/lib/esbonio/esbonio/lsp/testing.py b/lib/esbonio/esbonio/lsp/testing.py new file mode 100644 index 00000000..3ddf70b7 --- /dev/null +++ b/lib/esbonio/esbonio/lsp/testing.py @@ -0,0 +1,83 @@ +"""Utility functions to help with testing Language Server features.""" +import logging + +from typing import Optional, Set + +from pygls.types import Position +from pygls.workspace import Document + +logger = logging.getLogger(__name__) + + +def completion_test( + feature, text: str, expected: Optional[Set[str]], unexpected: Optional[Set[str]] +): + """Check to see if a feature provides the correct completion suggestions. + + **Only checking CompletionItem labels is supported** + + This function takes the given ``feature`` and calls it in the same manner as the + real language server so that it can simulate real usage without being a full blown + integration test. + + This requires ``suggest_triggers`` to be set and it to have a working ``suggest`` + method. + + Completions will be asked for with the cursor's position to be at the end of the + inserted ``text`` in a blank document by default. If your test case requires + additional context this can be included in ``text`` delimited by a ``\\f`` character. + + For example to pass text representing the following scenario (``^`` represents the + user's cursor):: + + .. image:: filename.png + :align: center + : + ^ + + The ``text`` parameter should be set to + ``.. image:: filename.png\\n :align: center\\n\\f :``. It's important to note that + newlines **cannot** come after the ``\\f`` character. + + If you want to test the case where no completions should be suggested, pass ``None`` + to both the ``expected`` and ``unexpected`` parameters. + + Parameters + ---------- + feature: + An instance of the language service feature to test. + text: + The text to offer completion suggestions for. + expected: + The set of completion item labels you expect to see in the output. + unexpected: + The set of completion item labels you do *not* expect to see in the output. + """ + + if "\f" in text: + contents, text = text.split("\f") + else: + contents = "" + + logger.debug("Context text: '%s'", contents) + logger.debug("Insertsion text: '%s'", text) + assert "\n" not in text, "Insertion text cannot contain newlines" + + document = Document("file:///test_doc.rst", contents) + position = Position(len(document.lines), len(text) - 1) + + results = [] + for trigger in feature.suggest_triggers: + match = trigger.match(text) + logger.debug("Match: %s", match) + + if match: + results += feature.suggest(match, document, position) + + items = {item.label for item in results} + + if expected is None: + assert len(items) == 0 + else: + assert expected == items & expected + assert set() == items & unexpected diff --git a/lib/esbonio/tests/conftest.py b/lib/esbonio/tests/conftest.py index f28c9a85..113cb987 100644 --- a/lib/esbonio/tests/conftest.py +++ b/lib/esbonio/tests/conftest.py @@ -88,13 +88,22 @@ def loader(filename, path_only=False): def sphinx(): """Return a Sphinx application instance pointed at the given project.""" + # Since extensions like intersphinx need to hit the network, let's cache + # app instances so we only incur this cost once. + cache = {} + basepath = pathlib.Path(__file__).parent / "data" def loader(project): src = str(basepath / project) + + if src in cache: + return cache[src] + build = str(basepath / project / "_build") - return Sphinx(src, src, build, build, "html", status=None, warning=None) + cache[src] = Sphinx(src, src, build, build, "html", status=None, warning=None) + return cache[src] return loader diff --git a/lib/esbonio/tests/data/sphinx-extensions/conf.py b/lib/esbonio/tests/data/sphinx-extensions/conf.py index 716017aa..6dd79b80 100644 --- a/lib/esbonio/tests/data/sphinx-extensions/conf.py +++ b/lib/esbonio/tests/data/sphinx-extensions/conf.py @@ -32,6 +32,9 @@ "sphinx": ("https://www.sphinx-doc.org/en/master", None), } +# Test with a different default domain. +primary_domain = "c" + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/lib/esbonio/tests/data/sphinx-extensions/theorems/pythagoras.rst b/lib/esbonio/tests/data/sphinx-extensions/theorems/pythagoras.rst index e6177ec1..8816b841 100644 --- a/lib/esbonio/tests/data/sphinx-extensions/theorems/pythagoras.rst +++ b/lib/esbonio/tests/data/sphinx-extensions/theorems/pythagoras.rst @@ -12,40 +12,40 @@ Implementation This project provides some functions which use Pythagoras' Theorem to calculate the length of a missing side of a right angled triangle when the other two are known. -.. module:: pythagoras +.. py:module:: pythagoras -.. currentmodule:: pythagoras +.. py:currentmodule:: pythagoras -.. data:: PI +.. py:data:: PI The value of the constant pi. -.. data:: UNKNOWN +.. py:data:: UNKNOWN Used to represent an unknown value. -.. class:: Triangle(a: float, b: float, c: float) +.. py:class:: Triangle(a: float, b: float, c: float) Represents a triangle - .. attribute:: a + .. py:attribute:: a The length of the side labelled ``a`` - .. attribute:: b + .. py:attribute:: b - The length of the side labelled ``b``` + The length of the side labelled ``b`` - .. attribute:: c + .. py:attribute:: c The length of the side labelled ``c`` - .. method:: is_right_angled() -> bool + .. py:method:: is_right_angled() -> bool :return: :code:`True` if the triangle is right angled. :rtype: bool -.. function:: calc_hypotenuse(a: float, b: float) -> float +.. py:function:: calc_hypotenuse(a: float, b: float) -> float Calculates the length of the hypotenuse of a right angled triangle. @@ -54,7 +54,7 @@ length of a missing side of a right angled triangle when the other two are known :return: Then length of the side ``c`` (the triangle's hypotenuse) :rtype: float -.. function:: calc_side(c: float, b: float) -> float +.. py:function:: calc_side(c: float, b: float) -> float Calculates the length of a side of a right angled triangle. diff --git a/lib/esbonio/tests/lsp/completion/test_directives.py b/lib/esbonio/tests/lsp/completion/test_directives.py deleted file mode 100644 index 0d9b1a27..00000000 --- a/lib/esbonio/tests/lsp/completion/test_directives.py +++ /dev/null @@ -1,46 +0,0 @@ -from mock import Mock - -import py.test - -from esbonio.lsp.completion.directives import DirectiveCompletion - - -@py.test.mark.parametrize( - "project,expected,unexpected", - [ - ( - "sphinx-default", - [ - "figure", - "function", - "glossary", - "image", - "list-table", - "module", - "toctree", - ], - [ - "testcode", - "autoclass", - "automodule", - "restructuredtext-test-directive", - ], - ) - ], -) -def test_discovery(sphinx, project, expected, unexpected): - """Ensure that we can discover directives to offer as completion suggestions""" - - rst = Mock() - rst.app = sphinx(project) - - completion = DirectiveCompletion(rst) - completion.discover() - - for name in expected: - message = "Missing directive '{}'" - assert name in completion.directives.keys(), message.format(name) - - for name in unexpected: - message = "Unexpected directive '{}'" - assert name not in completion.directives.keys(), message.format(name) diff --git a/lib/esbonio/tests/lsp/completion/test_integration.py b/lib/esbonio/tests/lsp/completion/test_integration.py index d1169a31..70cb2f0a 100644 --- a/lib/esbonio/tests/lsp/completion/test_integration.py +++ b/lib/esbonio/tests/lsp/completion/test_integration.py @@ -145,12 +145,7 @@ def do_completion_test( def role_target_patterns(rolename): return [ s.format(rolename) - for s in [ - ":{}:`", - ":{}:`More Info <", - " :{}:`", - " :{}:`Some Label <", - ] + for s in [":{}:`", ":{}:`More Info <", " :{}:`", " :{}:`Some Label <"] ] @@ -170,54 +165,9 @@ def intersphinx_patterns(rolename, namespace): @py.test.mark.parametrize( "text,setup", [ - *itertools.product( - [ - ".", - ".. doctest::", - ".. code-block::", - " .", - " .. doctest::", - " .. code-block::", - ".. _some_label:", - " .. _some_label:", - ], - [("sphinx-default", set())], - ), - *itertools.product( - [ - "..", - ".. ", - ".. d", - ".. code-b", - " ..", - " .. ", - " .. d", - " .. code-b", - ], - [ - ( - "sphinx-default", - {"admonition", "classmethod", "code-block", "image", "toctree"}, - ), - ( - "sphinx-extensions", - { - "admonition", - "classmethod", - "code-block", - "doctest", - "image", - "testsetup", - "toctree", - }, - ), - ], - ), *itertools.product( [":", ":r", "some text :", " :", " :r", " some text :"], - [ - ("sphinx-default", {"class", "doc", "func", "ref", "term"}), - ], + [("sphinx-default", {"class", "doc", "func", "ref", "term"}),], ), *itertools.product( role_target_patterns("class"), @@ -386,23 +336,3 @@ def test_expected_completions(client_server, testdata, text, setup): root = testdata(project, path_only=True) do_completion_test(client, server, root, "index.rst", text, expected) - - -def test_expected_directive_option_completions(client_server, testdata, caplog): - """Ensure that we can handle directive option completions.""" - - caplog.set_level(logging.INFO) - - client, server = client_server - root = testdata("sphinx-default", path_only=True) - expected = {"align", "alt", "class", "height", "name", "scale", "target", "width"} - - do_completion_test( - client, - server, - root, - "directive_options.rst", - " :a", - expected, - insert_newline=False, - ) diff --git a/lib/esbonio/tests/lsp/test_directives.py b/lib/esbonio/tests/lsp/test_directives.py new file mode 100644 index 00000000..8efcd5c8 --- /dev/null +++ b/lib/esbonio/tests/lsp/test_directives.py @@ -0,0 +1,185 @@ +import logging +import unittest.mock as mock + +import py.test + +from esbonio.lsp.directives import Directives +from esbonio.lsp.testing import completion_test + + +DEFAULT_EXPECTED = { + "function", + "module", + "option", + "program", + "image", + "toctree", + "c:macro", + "c:function", +} + +DEFAULT_UNEXPECTED = { + "autoclass", + "automodule", + "py:function", + "py:module", + "std:program", + "std:option", +} + +EXTENSIONS_EXPECTED = { + "py:function", + "py:module", + "option", + "program", + "image", + "toctree", + "macro", + "function", +} + +EXTENSIONS_UNEXPECTED = { + "autoclass", + "automodule", + "c:macro", + "module", + "std:program", + "std:option", +} + + +@py.test.mark.parametrize( + "project,text,expected,unexpected", + [ + ("sphinx-default", ".", None, None), + ("sphinx-default", "..", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", ".. ", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", ".. d", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", ".. code-b", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", ".. code-block::", None, None), + ("sphinx-default", ".. py:", None, None), + ( + "sphinx-default", + ".. c:", + {"c:macro", "c:function"}, + {"function", "image", "toctree"}, + ), + ("sphinx-default", ".. _some_label:", None, None), + ("sphinx-default", " .", None, None), + ("sphinx-default", " ..", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", " .. ", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", " .. d", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", " .. doctest::", None, None), + ("sphinx-default", " .. code-b", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", " .. code-block::", None, None), + ("sphinx-default", " .. py:", None, None), + ("sphinx-default", " .. _some_label:", None, None), + ( + "sphinx-default", + " .. c:", + {"c:macro", "c:function"}, + {"function", "image", "toctree"}, + ), + ("sphinx-extensions", ".", None, None), + ("sphinx-extensions", "..", EXTENSIONS_EXPECTED, EXTENSIONS_UNEXPECTED), + ("sphinx-extensions", ".. ", EXTENSIONS_EXPECTED, EXTENSIONS_UNEXPECTED), + ("sphinx-extensions", ".. d", EXTENSIONS_EXPECTED, EXTENSIONS_UNEXPECTED), + ("sphinx-extensions", ".. code-b", EXTENSIONS_EXPECTED, EXTENSIONS_UNEXPECTED), + ("sphinx-extensions", ".. code-block::", None, None), + ("sphinx-extensions", ".. _some_label:", None, None), + ( + "sphinx-extensions", + ".. py:", + {"py:function", "py:module"}, + {"image, toctree", "macro", "function"}, + ), + ("sphinx-extensions", ".. c:", None, None), + ("sphinx-extensions", " .", None, None), + ("sphinx-extensions", " ..", EXTENSIONS_EXPECTED, EXTENSIONS_UNEXPECTED), + ("sphinx-extensions", " .. ", EXTENSIONS_EXPECTED, EXTENSIONS_UNEXPECTED), + ("sphinx-extensions", " .. d", EXTENSIONS_EXPECTED, EXTENSIONS_UNEXPECTED), + ("sphinx-extensions", " .. doctest::", None, None), + ("sphinx-extensions", " .. _some_label:", None, None), + ( + "sphinx-extensions", + " .. code-b", + EXTENSIONS_EXPECTED, + EXTENSIONS_UNEXPECTED, + ), + ("sphinx-extensions", " .. code-block::", None, None), + ( + "sphinx-extensions", + ".. py:", + {"py:function", "py:module"}, + {"image, toctree", "macro", "function"}, + ), + ("sphinx-extensions", " .. c:", None, None), + ], +) +def test_directive_completions(sphinx, project, text, expected, unexpected): + """Ensure that we can provide the correct completions for directives.""" + + rst = mock.Mock() + rst.app = sphinx(project) + rst.logger = logging.getLogger("rst") + + feature = Directives(rst) + feature.initialize() + + completion_test(feature, text, expected, unexpected) + + +IMAGE_OPTS = {"align", "alt", "class", "height", "scale", "target", "width"} +PY_FUNC_OPTS = {"annotation", "async", "module", "noindex", "noindexentry"} +C_FUNC_OPTS = {"noindexentry"} + + +@py.test.mark.parametrize( + "project,text,expected,unexpected", + [ + ("sphinx-default", ".. image:: f.png\n\f :", IMAGE_OPTS, {"ref", "func"}), + ("sphinx-default", ".. function:: foo\n\f :", PY_FUNC_OPTS, {"ref", "func"}), + ( + "sphinx-default", + " .. image:: f.png\n\f :", + IMAGE_OPTS, + {"ref", "func"}, + ), + ( + "sphinx-default", + " .. function:: foo\n\f :", + PY_FUNC_OPTS, + {"ref", "func"}, + ), + ("sphinx-extensions", ".. image:: f.png\n\f :", IMAGE_OPTS, {"ref", "func"}), + ( + "sphinx-extensions", + ".. function:: foo\n\f :", + C_FUNC_OPTS, + {"ref", "func"}, + ), + ( + "sphinx-extensions", + " .. image:: f.png\n\f :", + IMAGE_OPTS, + {"ref", "func"}, + ), + ( + "sphinx-extensions", + " .. function:: foo\n\f :", + C_FUNC_OPTS, + {"ref", "func"}, + ), + ], +) +def test_directive_option_completions(sphinx, project, text, expected, unexpected): + """Ensure that we can provide the correct completions for directive options.""" + + rst = mock.Mock() + rst.app = sphinx(project) + rst.logger = logging.getLogger("rst") + + feature = Directives(rst) + feature.initialize() + + completion_test(feature, text, expected, unexpected) diff --git a/lib/esbonio/tests/lsp/completion/test_roles.py b/lib/esbonio/tests/lsp/test_roles.py similarity index 98% rename from lib/esbonio/tests/lsp/completion/test_roles.py rename to lib/esbonio/tests/lsp/test_roles.py index 25412a7b..b581550e 100644 --- a/lib/esbonio/tests/lsp/completion/test_roles.py +++ b/lib/esbonio/tests/lsp/test_roles.py @@ -4,7 +4,7 @@ from pygls.types import CompletionItemKind -from esbonio.lsp.completion.roles import RoleCompletion, RoleTargetCompletion +from esbonio.lsp.roles import RoleCompletion, RoleTargetCompletion @py.test.mark.parametrize( From 92019ac7db1a26c131d648c406f47ef24cc393f3 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Wed, 17 Feb 2021 23:54:58 +0000 Subject: [PATCH 02/41] Add domain support for role and role target completions (#104) - Merge `RoleCompletion` and `RoleTargetCompletion` classes into a single `Roles` class. - Add `dump` function that can easily convert an LSP item to its JSON representation for logging - Introduce common `get_domains` function in `esbonio.lsp.sphinx` that iterates through all registered domains. Works towards #71 --- .vscode/tasks.json | 7 + docs/contributing/lsp/directives.rst | 1 - docs/contributing/lsp/roles.rst | 4 + lib/esbonio/changes/104.feature.rst | 1 + lib/esbonio/esbonio/lsp/__init__.py | 16 +- lib/esbonio/esbonio/lsp/directives.py | 37 +- lib/esbonio/esbonio/lsp/roles.py | 427 +++++++++++------- lib/esbonio/esbonio/lsp/sphinx.py | 32 +- lib/esbonio/esbonio/lsp/testing.py | 4 + lib/esbonio/tests/conftest.py | 5 +- .../tests/lsp/completion/test_integration.py | 123 +---- lib/esbonio/tests/lsp/test_directives.py | 2 + lib/esbonio/tests/lsp/test_roles.py | 359 +++++++++------ 13 files changed, 584 insertions(+), 434 deletions(-) create mode 100644 docs/contributing/lsp/roles.rst create mode 100644 lib/esbonio/changes/104.feature.rst diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 71f3938e..4a5c1815 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -46,6 +46,13 @@ "cwd": "${workspaceRoot}/docs" } }, + { + "label": "pytest file", + "type": "shell", + "command": "source ${workspaceRoot}/.env/bin/activate && pytest ${file}", + "group": "test", + "problemMatcher": [], + }, { "label": "Grammar Tests", "type": "shell", diff --git a/docs/contributing/lsp/directives.rst b/docs/contributing/lsp/directives.rst index 8711b72a..2a774c5b 100644 --- a/docs/contributing/lsp/directives.rst +++ b/docs/contributing/lsp/directives.rst @@ -2,4 +2,3 @@ Directives ========== .. automodule:: esbonio.lsp.directives - :members: \ No newline at end of file diff --git a/docs/contributing/lsp/roles.rst b/docs/contributing/lsp/roles.rst new file mode 100644 index 00000000..b038661e --- /dev/null +++ b/docs/contributing/lsp/roles.rst @@ -0,0 +1,4 @@ +Roles +===== + +.. automodule:: esbonio.lsp.roles diff --git a/lib/esbonio/changes/104.feature.rst b/lib/esbonio/changes/104.feature.rst new file mode 100644 index 00000000..16af623a --- /dev/null +++ b/lib/esbonio/changes/104.feature.rst @@ -0,0 +1 @@ +Role and role target completions are now domain aware. diff --git a/lib/esbonio/esbonio/lsp/__init__.py b/lib/esbonio/esbonio/lsp/__init__.py index b2d3fe1f..490728e8 100644 --- a/lib/esbonio/esbonio/lsp/__init__.py +++ b/lib/esbonio/esbonio/lsp/__init__.py @@ -1,6 +1,6 @@ -# from __future__ import annotations - +import enum import importlib +import json import logging from typing import List, Optional @@ -99,6 +99,18 @@ def get_line_til_position(doc: Document, position: Position) -> str: return line[: position.character] +def dump(obj) -> str: + """Debug helper function that converts an object to JSON.""" + + def default(obj): + if isinstance(obj, enum.Enum): + return obj.value + + return {k: v for k, v in obj.__dict__.items() if v is not None} + + return json.dumps(obj, default=default) + + def create_language_server(modules: List[str]) -> RstLanguageServer: """Create a new language server instance. diff --git a/lib/esbonio/esbonio/lsp/directives.py b/lib/esbonio/esbonio/lsp/directives.py index e28121e5..a3b29109 100644 --- a/lib/esbonio/esbonio/lsp/directives.py +++ b/lib/esbonio/esbonio/lsp/directives.py @@ -17,11 +17,14 @@ from pygls.workspace import Document from esbonio.lsp import RstLanguageServer, LanguageFeature +from esbonio.lsp.sphinx import get_domains + DIRECTIVE = re.compile(r"\s*\.\.[ ](?P[\w]+:)?(?P[\w-]+)::") """A regular expression that matches a complete, valid directive declaration. Not including the arguments or options.""" + PARTIAL_DIRECTIVE = re.compile( r""" (?P^\s*) # directives can be indented @@ -36,6 +39,7 @@ """A regular expression that matches a partial directive declaraiton. Used when generating auto complete suggestions.""" + PARTIAL_DIRECTIVE_OPTION = re.compile( r""" (?P\s+) # directive options must only be preceeded by whitespace @@ -60,32 +64,19 @@ def discover(self): ignored_directives = ["restructuredtext-test-directive"] # Find directives that have been registered directly with docutils. - dirs = {**directives._directive_registry, **directives._directives} + found_directives = {**directives._directive_registry, **directives._directives} # Find directives under Sphinx domains - if self.rst.app is not None: - - domains = self.rst.app.registry.domains - primary_domain = self.rst.app.config.primary_domain - - for name, domain in domains.items(): - namefmt = "{name}:{dirname}" - - # The "standard" domain and the "primary_domain" do not require - # the prefix - if name == "std" or name == primary_domain: - namefmt = "{dirname}" + for prefix, domain in get_domains(self.rst.app): + fmt = "{prefix}:{name}" if prefix else "{name}" - dirs.update( - { - namefmt.format(name=name, dirname=dirname): directive - for dirname, directive in domain.directives.items() - } - ) + for name, directive in domain.directives.items(): + key = fmt.format(name=name, prefix=prefix) + found_directives[key] = directive self.directives = { k: self.resolve_directive(v) - for k, v in dirs.items() + for k, v in found_directives.items() if k not in ignored_directives } @@ -146,7 +137,7 @@ def suggest_options( groups = match.groupdict() - self.logger.debug("Suggesting options") + self.logger.info("Suggesting options") self.logger.debug("Match groups: %s", groups) indent = groups["indent"] @@ -171,7 +162,6 @@ def suggest_options( domain = match.group("domain") or "" name = f"{domain}{match.group('name')}" - self.logger.debug("Returning options for directive: %s", name) return self.options.get(name, []) def directive_to_completion_item( @@ -225,7 +215,8 @@ def directive_to_completion_item( Parameters ---------- name: - The name of the directive as a user would type in an rst document + The name of the directive as a user would type in an reStructuredText + document directive: The class definition that implements the Directive's behavior match: diff --git a/lib/esbonio/esbonio/lsp/roles.py b/lib/esbonio/esbonio/lsp/roles.py index 98f710db..bea888c1 100644 --- a/lib/esbonio/esbonio/lsp/roles.py +++ b/lib/esbonio/esbonio/lsp/roles.py @@ -1,28 +1,99 @@ -"""Role completions.""" +"""Role support.""" import re from typing import Dict, List from docutils.parsers.rst import roles -from pygls.types import CompletionItem, CompletionItemKind, DidSaveTextDocumentParams +from pygls.types import ( + CompletionItem, + CompletionItemKind, + DidSaveTextDocumentParams, + Position, + Range, + TextEdit, +) +from pygls.workspace import Document from sphinx.domains import Domain -from esbonio.lsp import RstLanguageServer +from esbonio.lsp import RstLanguageServer, LanguageFeature, dump from esbonio.lsp.directives import DIRECTIVE +from esbonio.lsp.sphinx import get_domains + + +PARTIAL_ROLE = re.compile( + r""" + (^|.*[ ]) # roles must be preceeded by a space, or start the line + (?P: # roles start with the ':' character + (?!:) # make sure the next character is not ':' + (?P[\w]+:)? # there may be a domain namespace + (?P[\w-]*)) # match the role name + $ # ensure pattern only matches incomplete roles + """, + re.MULTILINE | re.VERBOSE, +) +"""A regular expression that matches a partial role. + +For example:: + + :re + +Used when generating auto complete suggestions. +""" + + +PARTIAL_PLAIN_TARGET = re.compile( + r""" + (^|.*[ ]) # roles must be preceeded by a space, or start the line + (?P: # roles start with the ':' character + (?!:) # make sure the next character is not ':' + (?P[\w]+:)? # there may be a domain namespace + (?P[\w-]*) # followed by the role name + :) # the role name ends with a ':' + ` # the target begins with a '`' + (?P[^<:`]*) # match "plain link" targets + $ + """, + re.MULTILINE | re.VERBOSE, +) +"""A regular expression that matches a partial "plain" role target. + +For example:: + + :ref:`som + +Used when generating auto complete suggestions. +""" + +PARTIAL_ALIASED_TARGET = re.compile( + r""" + (^|.*[ ]) # roles must be preceeded by a space, or start the line + (?P: # roles start with the ':' character + (?!:) # make sure the next character is not ':' + (?P[\w]+:)? # there may be a domain namespace + (?P[\w-]*) # followed by the role name + :) # the role name ends with a ':' + ` # the target begins with a '`'` + .*< # the actual target name starts after a '<' + (?P[^`:]*) # match "aliased" targets + $ + """, + re.MULTILINE | re.VERBOSE, +) +"""A regular expression that matches an "aliased" role target. + +For example:: + + :ref:`More info CompletionItem: return CompletionItem( - namespace, detail="intersphinx namespace", kind=CompletionItemKind.Module, - ) - - -def role_to_completion_item(name, role) -> CompletionItem: - return CompletionItem( - name, - kind=CompletionItemKind.Function, - detail="role", - insert_text="{}:".format(name), + namespace, + detail="intersphinx namespace", + kind=CompletionItemKind.Module, ) @@ -46,11 +117,6 @@ def role_to_completion_item(name, role) -> CompletionItem: } -def target_to_completion_item(name, display, type_) -> CompletionItem: - kind = TARGET_KINDS.get(type_, CompletionItemKind.Reference) - return CompletionItem(name, kind=kind, detail=str(display), insert_text=name) - - def intersphinx_target_to_completion_item(label, item, type_) -> CompletionItem: kind = TARGET_KINDS.get(type_, CompletionItemKind.Reference) source, version, _, display = item @@ -66,66 +132,121 @@ def intersphinx_target_to_completion_item(label, item, type_) -> CompletionItem: return CompletionItem(label, kind=kind, detail=detail, insert_text=label) -class RoleCompletion: - """Completion handler for roles.""" - - def __init__(self, rst: RstLanguageServer): - self.rst = rst +class Roles(LanguageFeature): + """Role support for the language server.""" def initialize(self): - self.discover() + self.discover_roles() + self.discover_targets() + + def save(self, params: DidSaveTextDocumentParams): + self.discover_targets() + + def discover_roles(self): + """Look up for valid role defintions to to offer as autocomplete suggestions. + + *This method only needs to be called once per application instance.* + + This will look for all the roles registered with docutils as well as the + roles that are stored on a Sphinx domain object. + + Additionally, while we are looping through the domain objects, we construct + the ``target_types`` dictionary. This is used when providing role target + completions by giving the list of object types the current role is able to + link with. - def discover(self): + For example, consider the :rst:role:`sphinx:py:func` and + :rst:role:`sphinx:py:class` roles from the Python domain. As the ``func`` role + links to Python functions and the ``class`` role links to Python classes *and* + exceptions we would end up with + + .. code-block:: python + + { + "func": ["py:function"], + "class": ["py:class", "py:exception"] + } + + """ # Find roles that have been registered directly with docutils - local_roles = { - k: v for k, v in roles._roles.items() if v != roles.unimplemented_role - } - role_registry = { - k: v - for k, v in roles._role_registry.items() - if v != roles.unimplemented_role + self.target_types = {} + found_roles = {**roles._roles, **roles._role_registry} + + # Find roles under Sphinx domains + for prefix, domain in get_domains(self.rst.app): + fmt = "{prefix}:{name}" if prefix else "{name}" + + for name, role in domain.roles.items(): + key = fmt.format(name=name, prefix=prefix) + found_roles[key] = role + + # Also build a map we can use when looking up target completions. + for name, item_type in domain.object_types.items(): + for role in item_type.roles: + key = fmt.format(name=role, prefix=prefix) + target_types = self.target_types.get(key, None) + + if target_types is None: + target_types = [] + + target_types.append(fmt.format(name=name, prefix=prefix)) + self.target_types[key] = target_types + + self.roles = { + k: v for k, v in found_roles.items() if v != roles.unimplemented_role } - std_roles = {} - py_roles = {} - if self.rst.app is not None: + self.logger.info("Discovered %s roles", len(self.roles)) + self.logger.info("Discovered %s target types", len(self.target_types)) - # Find roles that are held in a Sphinx domain. - # TODO: Implement proper domain handling, will focus on std+python for now - domains = self.rst.app.registry.domains - std_roles = domains["std"].roles - py_roles = domains["py"].roles + self.logger.debug(self.roles.keys()) + self.logger.debug(self.target_types) - rs = {**local_roles, **role_registry, **std_roles, **py_roles} + def discover_targets(self): + """Look up all the targets we can offer as autocomplete suggestions. - self.roles = {k: role_to_completion_item(k, v) for k, v in rs.items()} - self.rst.logger.debug("Discovered %s roles", len(self.roles)) + *This method needs to be called each time a document has been saved.* + """ + self.target_objects = {} - suggest_triggers = [ - re.compile( - r""" - (^|.*[ ]) # roles must be preceeded by a space, or start the line - : # roles start with the ':' character - (?!:) # make sure the next character is not ':' - [\w-]* # match the role name - $ # ensure pattern only matches incomplete roles - """, - re.MULTILINE | re.VERBOSE, - ) - ] + for prefix, domain in get_domains(self.rst.app): + fmt = "{prefix}:{name}" if prefix else "{name}" - def suggest(self, match, doc, position) -> List[CompletionItem]: + for (name, display_name, obj_type, _, _, _) in domain.get_objects(): + key = fmt.format(name=obj_type, prefix=prefix) + items = self.target_objects.get(key, None) + + if items is None: + items = [] + self.target_objects[key] = items + + items.append( + self.target_object_to_completion_item(name, display_name, obj_type) + ) + + suggest_triggers = [PARTIAL_ROLE, PARTIAL_PLAIN_TARGET, PARTIAL_ALIASED_TARGET] + + def suggest( + self, match: "re.Match", doc: Document, position: Position + ) -> List[CompletionItem]: indent = match.group(1) + if "target" in match.groupdict(): + return self.suggest_targets(match, position) + # If there's no indent, then this can only be a role defn if indent == "": - return list(self.roles.values()) + return self.suggest_roles(match, position) # Otherwise, search backwards until we find a blank line or an unindent # so that we can determine the appropriate context. linum = position.line - 1 - line = doc.lines[linum] + + try: + line = doc.lines[linum] + except IndexError: + return self.suggest_roles(match, position) while line.startswith(indent): linum -= 1 @@ -136,88 +257,116 @@ def suggest(self, match, doc, position) -> List[CompletionItem]: if DIRECTIVE.match(line): return [] - return list(self.roles.values()) + return self.suggest_roles(match, position) + def suggest_roles( + self, match: "re.Match", position: Position + ) -> List[CompletionItem]: + self.logger.info("Suggesting roles") -def build_role_target_map(domain: Domain) -> Dict[str, List[str]]: - """Return a map of role names to the objects they link to. + domain = match.groupdict()["domain"] or "" + items = [] - Parameters - ---------- - domain: - The Sphinx domain to build the target map for. - """ - types = {} + for name, role in self.roles.items(): - for name, obj in domain.object_types.items(): - for role in obj.roles: - objs = types.get(role, None) + if not name.startswith(domain): + continue - if objs is None: - objs = [] + item = self.role_to_completion_item(name, role, match, position) + items.append(item) - objs.append(name) - types[role] = objs + return items - return types + def suggest_targets( + self, match: "re.Match", position: Position + ) -> List[CompletionItem]: + self.logger.info("Suggesting targets") -def build_target_map(domain: Domain) -> Dict[str, List[CompletionItem]]: - """Return a map of object types to a list of completion items.""" - completion_items = {} + groups = match.groupdict() + domain = groups["domain"] or "" + key = f"{domain}{groups['name']}" - for (name, disp, type_, _, _, _) in domain.get_objects(): - items = completion_items.get(type_, None) + object_types = self.target_types.get(key, None) - if items is None: - items = [] - completion_items[type_] = items + self.logger.debug("Getting suggestions for '%s'", key) + self.logger.debug("Role targets object types: %s", object_types) - items.append(target_to_completion_item(name, disp, type_)) + if object_types is None: + return [] - return completion_items + targets = [] + for type_ in object_types: + targets += self.target_objects.get(type_, []) + return targets -class RoleTargetCompletion: - """Completion handler for role targets.""" + def role_to_completion_item( + self, name: str, role, match: "re.Match", position: Position + ) -> CompletionItem: + """Convert an rst role to its CompletionItem representation. + + With domain support it's necessary to compute the CompletionItem representation + specifically for each completion site. See + :meth:`~esbonio.lsp.directives.Directives.directive_to_completion_item` for + more historical information. + + For some reason, even though these completion items are constructed in the same + manner as the ones for directives using them in VSCode does not feel as nice.... + + Parameters + ---------- + name: + The name of the role as a user would type into an reStructuredText document. + role: + The implementation of the role. + match: + The regular expression match object that represents the line we are providing + the autocomplete suggestions for. + position: + The position in the source code where the autocompletion request was sent + from. + """ + + groups = match.groupdict() + + line = position.line + start = position.character - len(groups["role"]) + end = position.character + + insert_text = f":{name}:" + + item = CompletionItem( + name, + kind=CompletionItemKind.Function, + filter_text=insert_text, + detail="role", + text_edit=TextEdit( + range=Range(Position(line, start), Position(line, end)), + new_text=insert_text, + ), + ) - def __init__(self, rst: RstLanguageServer): - self.rst = rst + self.logger.debug("Item %s", dump(item)) + return item - def initialize(self): - self.discover_target_types() - self.discover_targets() + def target_object_to_completion_item( + self, name: str, display_name: str, obj_type: str + ) -> CompletionItem: + """Convert a target object to its CompletionItem representation.""" - def save(self, params: DidSaveTextDocumentParams): - self.discover_targets() + kind = TARGET_KINDS.get(obj_type, CompletionItemKind.Reference) + + return CompletionItem( + name, kind=kind, detail=str(display_name), insert_text=name + ) - suggest_triggers = [ - re.compile( - r""" - (^|.*[ ]) # roles must be preceeded by a space, or start the line - : # roles start with the ':' character - (?P[\w-]+) # capture the role name, suggestions will change based on it - : # the role name ends with a ':' - ` # the target begins with a '`' - (?P[^<:`]*) # match "plain link" targets - $ - """, - re.MULTILINE | re.VERBOSE, - ), - re.compile( - r""" - (^|.*[ ]) # roles must be preceeded by a space, or start the line - : # roles start with the ':' character - (?P[\w-]+) # capture the role name, suggestions will change based on it - : # the role name ends with a ':' - ` # the target begins with a '`'` - .*< # the actual target name starts after a '<' - (?P[^`:]*) # match "aliased" targets - $ - """, - re.MULTILINE | re.VERBOSE, - ), - ] + +class RoleTargets: + """Role target support for the language server.""" + + def __init__(self, rst: RstLanguageServer): + self.rst = rst def suggest(self, match, doc, position) -> List[CompletionItem]: # TODO: Detect if we're in an angle bracket e.g. :ref:`More Info <|` in that @@ -238,31 +387,6 @@ def suggest(self, match, doc, position) -> List[CompletionItem]: return targets - def discover_target_types(self): - - if self.rst.app is None: - return - - # TODO: Implement proper domain handling, will focus on std+python for now - domains = self.rst.app.env.domains - py = domains["py"] - std = domains["std"] - - self.target_types = {**build_role_target_map(py), **build_role_target_map(std)} - - def discover_targets(self): - - if self.rst.app is None: - self.targets = {} - return - - # TODO: Implement proper domain handling, will focus on std+python for now - domains = self.rst.app.env.domains - py = domains["py"] - std = domains["std"] - - self.targets = {**build_target_map(py), **build_target_map(std)} - class InterSphinxNamespaceCompletion: """Completion handler for intersphinx namespaces.""" @@ -281,7 +405,7 @@ def initialize(self): "Discovered %s intersphinx namespaces", len(self.namespaces) ) - suggest_triggers = RoleTargetCompletion.suggest_triggers + # suggest_triggers = RoleTargetCompletion.suggest_triggers def suggest(self, match, doc, position) -> List[CompletionItem]: return list(self.namespaces.values()) @@ -384,12 +508,5 @@ def suggest(self, match, doc, position) -> List[CompletionItem]: def setup(rst: RstLanguageServer): - role_completion = RoleCompletion(rst) - role_target_completion = RoleTargetCompletion(rst) - intersphinx_namespaces = InterSphinxNamespaceCompletion(rst) - intersphinx_targets = InterSphinxTargetCompletion(rst) - + role_completion = Roles(rst) rst.add_feature(role_completion) - rst.add_feature(role_target_completion) - rst.add_feature(intersphinx_namespaces) - rst.add_feature(intersphinx_targets) diff --git a/lib/esbonio/esbonio/lsp/sphinx.py b/lib/esbonio/esbonio/lsp/sphinx.py index 4e7bab69..a1d3899e 100644 --- a/lib/esbonio/esbonio/lsp/sphinx.py +++ b/lib/esbonio/esbonio/lsp/sphinx.py @@ -3,7 +3,7 @@ import pathlib import re -from typing import Optional +from typing import Iterator, Optional, Tuple from urllib.parse import urlparse, unquote import appdirs @@ -17,6 +17,7 @@ Range, ) from sphinx.application import Sphinx +from sphinx.domains import Domain from sphinx.util import console from esbonio.lsp import RstLanguageServer @@ -30,12 +31,41 @@ ) """Regular Expression used to identify warnings/errors in Sphinx's output.""" + PROBLEM_SEVERITY = { "WARNING": DiagnosticSeverity.Warning, "ERROR": DiagnosticSeverity.Error, } +def get_domains(app: Sphinx) -> Iterator[Tuple[str, Domain]]: + """Get all the domains registered with an applications. + + Returns a generator that iterates through all of an application's domains, + taking into account configuration variables such as ``primary_domain``. + Yielded values will be a tuple of the form ``(prefix, domain)`` where + + - ``prefix`` is the namespace that should be used when referencing items + in the domain + - ``domain`` is the domain object itself. + """ + + if app is None: + return [] + + domains = app.env.domains + primary_domain = app.config.primary_domain + + for name, domain in domains.items(): + prefix = name + + # Items from the standard and primary domains don't require the namespace prefix + if name == "std" or name == primary_domain: + prefix = "" + + yield prefix, domain + + def find_conf_py(root_uri: str) -> Optional[pathlib.Path]: """Attempt to find Sphinx's configuration file in the given workspace.""" diff --git a/lib/esbonio/esbonio/lsp/testing.py b/lib/esbonio/esbonio/lsp/testing.py index 3ddf70b7..c0c8a1fa 100644 --- a/lib/esbonio/esbonio/lsp/testing.py +++ b/lib/esbonio/esbonio/lsp/testing.py @@ -76,6 +76,10 @@ def completion_test( items = {item.label for item in results} + logger.debug("Results: %s", items) + logger.debug("Expected: %s", expected) + logger.debug("Unexpected: %s", unexpected) + if expected is None: assert len(items) == 0 else: diff --git a/lib/esbonio/tests/conftest.py b/lib/esbonio/tests/conftest.py index 113cb987..10cd9bb5 100644 --- a/lib/esbonio/tests/conftest.py +++ b/lib/esbonio/tests/conftest.py @@ -102,7 +102,10 @@ def loader(project): build = str(basepath / project / "_build") - cache[src] = Sphinx(src, src, build, build, "html", status=None, warning=None) + app = Sphinx(src, src, build, build, "html", status=None, warning=None) + app.builder.read() + + cache[src] = app return cache[src] return loader diff --git a/lib/esbonio/tests/lsp/completion/test_integration.py b/lib/esbonio/tests/lsp/completion/test_integration.py index 70cb2f0a..e8c4b18f 100644 --- a/lib/esbonio/tests/lsp/completion/test_integration.py +++ b/lib/esbonio/tests/lsp/completion/test_integration.py @@ -1,4 +1,3 @@ -import logging import itertools import pathlib import time @@ -142,13 +141,6 @@ def do_completion_test( assert len(missing) == 0, "Missing expected items, {}".format(missing) -def role_target_patterns(rolename): - return [ - s.format(rolename) - for s in [":{}:`", ":{}:`More Info <", " :{}:`", " :{}:`Some Label <"] - ] - - def intersphinx_patterns(rolename, namespace): return [ s.format(rolename, namespace) @@ -161,123 +153,10 @@ def intersphinx_patterns(rolename, namespace): ] -@py.test.mark.integration +@py.test.mark.skip @py.test.mark.parametrize( "text,setup", [ - *itertools.product( - [":", ":r", "some text :", " :", " :r", " some text :"], - [("sphinx-default", {"class", "doc", "func", "ref", "term"}),], - ), - *itertools.product( - role_target_patterns("class"), - [ - ("sphinx-default", {"pythagoras.Triangle"}), - ("sphinx-extensions", {"pythagoras.Triangle", "python", "sphinx"}), - ], - ), - *itertools.product( - role_target_patterns("doc"), - [ - ( - "sphinx-default", - {"index", "glossary", "theorems/index", "theorems/pythagoras"}, - ), - ( - "sphinx-extensions", - { - "index", - "glossary", - "python", - "sphinx", - "theorems/index", - "theorems/pythagoras", - }, - ), - ], - ), - *itertools.product( - role_target_patterns("func"), - [ - ( - "sphinx-default", - {"pythagoras.calc_hypotenuse", "pythagoras.calc_side"}, - ), - ( - "sphinx-extensions", - { - "pythagoras.calc_hypotenuse", - "pythagoras.calc_side", - "python", - "sphinx", - }, - ), - ], - ), - *itertools.product( - role_target_patterns("meth"), - [ - ("sphinx-default", {"pythagoras.Triangle.is_right_angled"}), - ( - "sphinx-extensions", - {"pythagoras.Triangle.is_right_angled", "python", "sphinx"}, - ), - ], - ), - *itertools.product( - role_target_patterns("obj"), - [ - ( - "sphinx-default", - { - "pythagoras.Triangle", - "pythagoras.Triangle.is_right_angled", - "pythagoras.calc_hypotenuse", - "pythagoras.calc_side", - }, - ), - ( - "sphinx-extensions", - { - "pythagoras.Triangle", - "pythagoras.Triangle.is_right_angled", - "pythagoras.calc_hypotenuse", - "pythagoras.calc_side", - "python", - "sphinx", - }, - ), - ], - ), - *itertools.product( - role_target_patterns("ref"), - [ - ( - "sphinx-default", - { - "genindex", - "modindex", - "py-modindex", - "pythagoras_theorem", - "search", - "welcome", - }, - ), - ( - "sphinx-extensions", - { - "genindex", - "modindex", - "py-modindex", - "pythagoras_theorem", - "python", - "sphinx", - "search", - "welcome", - }, - ), - ], - ), *itertools.product( intersphinx_patterns("ref", "python"), [ diff --git a/lib/esbonio/tests/lsp/test_directives.py b/lib/esbonio/tests/lsp/test_directives.py index 8efcd5c8..533cb615 100644 --- a/lib/esbonio/tests/lsp/test_directives.py +++ b/lib/esbonio/tests/lsp/test_directives.py @@ -25,6 +25,7 @@ "py:module", "std:program", "std:option", + "restructuredtext-test-directive", } EXTENSIONS_EXPECTED = { @@ -45,6 +46,7 @@ "module", "std:program", "std:option", + "restructuredtext-test-directive", } diff --git a/lib/esbonio/tests/lsp/test_roles.py b/lib/esbonio/tests/lsp/test_roles.py index b581550e..c6118c92 100644 --- a/lib/esbonio/tests/lsp/test_roles.py +++ b/lib/esbonio/tests/lsp/test_roles.py @@ -1,164 +1,265 @@ -from mock import Mock +import itertools +import logging +import unittest.mock as mock import py.test from pygls.types import CompletionItemKind -from esbonio.lsp.roles import RoleCompletion, RoleTargetCompletion +from esbonio.lsp.roles import Roles +from esbonio.lsp.testing import completion_test +C_EXPECTED = {"c:func", "c:macro"} +C_UNEXPECTED = {"ref", "doc", "py:func", "py:mod"} -@py.test.mark.parametrize( - "project,expected,unexpected", - [ - ( - "sphinx-default", - [ - "emphasis", - "subscript", - "raw", - "func", - "meth", - "class", - "ref", - "doc", - "term", - ], - ["named-reference", "restructuredtext-unimplemented-role"], - ) - ], -) -def test_role_discovery(sphinx, project, expected, unexpected): - """Ensure that we can correctly discover role definitions to offer as - suggestions.""" - - rst = Mock() - rst.app = sphinx(project) +DEFAULT_EXPECTED = {"doc", "func", "mod", "ref", "c:func"} +DEFAULT_UNEXPECTED = {"py:func", "py:mod", "restructuredtext-unimplemented-role"} - completion = RoleCompletion(rst) - completion.discover() +EXT_EXPECTED = {"doc", "py:func", "py:mod", "ref", "func"} +EXT_UNEXPECTED = {"c:func", "c:macro", "restructuredtext-unimplemented-role"} - for name in expected: - message = "Missing expected role '{}'" - assert name in completion.roles.keys(), message.format(name) - - for name in unexpected: - message = "Unexpected role '{}'" - assert name not in completion.roles.keys(), message.format(name) +PY_EXPECTED = {"py:func", "py:mod"} +PY_UNEXPECTED = {"ref", "doc", "c:func", "c:macro"} @py.test.mark.parametrize( - "role,objects", + "project,text,expected,unexpected", [ - ("attr", {"attribute"}), - ("class", {"class", "exception"}), - ("data", {"data"}), - ("doc", {"doc"}), - ("envvar", {"envvar"}), - ("exc", {"class", "exception"}), - ("func", {"function"}), - ("meth", {"method", "classmethod", "staticmethod"}), - ( - "obj", - { - "attribute", - "class", - "classmethod", - "data", - "exception", - "function", - "method", - "module", - "staticmethod", - }, - ), - ("ref", {"label"}), - ("term", {"term"}), + ("sphinx-default", ":", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", ":r", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", ":ref:", None, None), + ("sphinx-default", ":py:", None, None), + ("sphinx-default", ":c:", C_EXPECTED, C_UNEXPECTED), + ("sphinx-default", "some text :", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", "some text :ref:", None, None), + ("sphinx-default", "some text :py:", None, None), + ("sphinx-default", "some text :c:", C_EXPECTED, C_UNEXPECTED), + ("sphinx-default", " :", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", " :r", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", " :ref:", None, None), + ("sphinx-default", " :py:", None, None), + ("sphinx-default", " :c:", C_EXPECTED, C_UNEXPECTED), + ("sphinx-default", " some text :", DEFAULT_EXPECTED, DEFAULT_UNEXPECTED), + ("sphinx-default", " some text :ref:", None, None), + ("sphinx-default", " some text :py:", None, None), + ("sphinx-default", " some text :c:", C_EXPECTED, C_UNEXPECTED), + ("sphinx-extensions", ":", EXT_EXPECTED, EXT_UNEXPECTED), + ("sphinx-extensions", ":r", EXT_EXPECTED, EXT_UNEXPECTED), + ("sphinx-extensions", ":ref:", None, None), + ("sphinx-extensions", ":py:", PY_EXPECTED, PY_UNEXPECTED), + ("sphinx-extensions", ":c:", None, None), + ("sphinx-extensions", "some text :", EXT_EXPECTED, EXT_UNEXPECTED), + ("sphinx-extensions", "some text :ref:", None, None), + ("sphinx-extensions", "some text :py:", PY_EXPECTED, PY_UNEXPECTED), + ("sphinx-extensions", "some text :c:", None, None), + ("sphinx-extensions", " :", EXT_EXPECTED, EXT_UNEXPECTED), + ("sphinx-extensions", " :r", EXT_EXPECTED, EXT_UNEXPECTED), + ("sphinx-extensions", " :ref:", None, None), + ("sphinx-extensions", " :py:", PY_EXPECTED, PY_UNEXPECTED), + ("sphinx-extensions", " :c:", None, None), + ("sphinx-extensions", " some text :", EXT_EXPECTED, EXT_UNEXPECTED), + ("sphinx-extensions", " some text :ref:", None, None), + ("sphinx-extensions", " some text :py:", PY_EXPECTED, PY_UNEXPECTED), + ("sphinx-extensions", " some text :c:", None, None), ], ) -def test_target_type_discovery(sphinx, role, objects): - """Ensure that we can correctly map roles to their correspondig object types.""" +def test_role_completions(sphinx, project, text, expected, unexpected): + """Ensure that we can offer correct role suggestions.""" - rst = Mock() - rst.app = sphinx("sphinx-default") + rst = mock.Mock() + rst.app = sphinx(project) + rst.logger = logging.getLogger("rst") + + feature = Roles(rst) + feature.initialize() + + completion_test(feature, text, expected, unexpected) - completion = RoleTargetCompletion(rst) - completion.discover_target_types() - assert {*completion.target_types[role]} == objects +def role_target_patterns(name): + return [ + s.format(name) + for s in [":{}:`", ":{}:`More Info <", " :{}:`", " :{}:`Some Label <"] + ] @py.test.mark.parametrize( - "project,type,kind,expected", + "text,setup", [ - ( - "sphinx-default", - "attribute", - CompletionItemKind.Field, - {"pythagoras.Triangle.a", "pythagoras.Triangle.b", "pythagoras.Triangle.c"}, + # Standard domain + *itertools.product( + role_target_patterns("doc"), + [ + ( + "sphinx-default", + {"index", "glossary", "theorems/index", "theorems/pythagoras"}, + set(), + ) + ], ), - ("sphinx-default", "class", CompletionItemKind.Class, {"pythagoras.Triangle"}), - ( - "sphinx-default", - "doc", - CompletionItemKind.File, - { - "glossary", - "index", - "theorems/index", - "theorems/pythagoras", - "directive_options", - }, + *itertools.product( + role_target_patterns("ref"), + [ + ( + "sphinx-default", + { + "genindex", + "modindex", + "py-modindex", + "pythagoras_theorem", + "search", + "welcome", + }, + set(), + ) + ], ), - ( - "sphinx-default", - "envvar", - CompletionItemKind.Variable, - {"ANGLE_UNIT", "PRECISION"}, + # Python Domain + *itertools.product( + role_target_patterns("class"), + [ + ("sphinx-default", {"pythagoras.Triangle"}, set()), + ("sphinx-extensions", set(), {"pythagoras.Triangle"}), + ], ), - ( - "sphinx-default", - "function", - CompletionItemKind.Function, - {"pythagoras.calc_side", "pythagoras.calc_hypotenuse"}, + *itertools.product( + role_target_patterns("py:class"), + [ + ("sphinx-default", set(), {"pythagoras.Triangle"}), + ("sphinx-extensions", {"pythagoras.Triangle"}, set()), + ], + ), + *itertools.product( + role_target_patterns("func"), + [ + ( + "sphinx-default", + {"pythagoras.calc_hypotenuse", "pythagoras.calc_side"}, + set(), + ), + ( + "sphinx-extensions", + set(), + {"pythagoras.calc_hypotenuse", "pythagoras.calc_side"}, + ), + ], + ), + *itertools.product( + role_target_patterns("py:func"), + [ + ( + "sphinx-default", + set(), + {"pythagoras.calc_hypotenuse", "pythagoras.calc_side"}, + ), + ( + "sphinx-extensions", + {"pythagoras.calc_hypotenuse", "pythagoras.calc_side"}, + set(), + ), + ], + ), + *itertools.product( + role_target_patterns("meth"), + [ + ("sphinx-default", {"pythagoras.Triangle.is_right_angled"}, set()), + ("sphinx-extensions", set(), {"pythagoras.Triangle.is_right_angled"}), + ], ), - ( - "sphinx-default", - "method", - CompletionItemKind.Method, - {"pythagoras.Triangle.is_right_angled"}, + *itertools.product( + role_target_patterns("py:meth"), + [ + ("sphinx-default", set(), {"pythagoras.Triangle.is_right_angled"}), + ("sphinx-extensions", {"pythagoras.Triangle.is_right_angled"}, set()), + ], ), - ("sphinx-default", "module", CompletionItemKind.Module, {"pythagoras"}), - ( - "sphinx-default", - "label", - CompletionItemKind.Reference, - { - "genindex", - "modindex", - "py-modindex", - "pythagoras_theorem", - "search", - "welcome", - }, + *itertools.product( + role_target_patterns("obj"), + [ + ( + "sphinx-default", + { + "pythagoras", + "pythagoras.PI", + "pythagoras.UNKNOWN", + "pythagoras.Triangle", + "pythagoras.Triangle.a", + "pythagoras.Triangle.b", + "pythagoras.Triangle.c", + "pythagoras.Triangle.is_right_angled", + "pythagoras.calc_hypotenuse", + "pythagoras.calc_side", + }, + set(), + ), + ( + "sphinx-extensions", + set(), + { + "pythagoras", + "pythagoras.PI", + "pythagoras.UNKNOWN", + "pythagoras.Triangle", + "pythagoras.Triangle.a", + "pythagoras.Triangle.b", + "pythagoras.Triangle.c", + "pythagoras.Triangle.is_right_angled", + "pythagoras.calc_hypotenuse", + "pythagoras.calc_side", + }, + ), + ], ), - ( - "sphinx-default", - "term", - CompletionItemKind.Text, - {"hypotenuse", "right angle"}, + *itertools.product( + role_target_patterns("py:obj"), + [ + ( + "sphinx-extensions", + { + "pythagoras", + "pythagoras.PI", + "pythagoras.UNKNOWN", + "pythagoras.Triangle", + "pythagoras.Triangle.a", + "pythagoras.Triangle.b", + "pythagoras.Triangle.c", + "pythagoras.Triangle.is_right_angled", + "pythagoras.calc_hypotenuse", + "pythagoras.calc_side", + }, + set(), + ), + ( + "sphinx-default", + set(), + { + "pythagoras", + "pythagoras.PI", + "pythagoras.UNKNOWN", + "pythagoras.Triangle", + "pythagoras.Triangle.a", + "pythagoras.Triangle.b", + "pythagoras.Triangle.c", + "pythagoras.Triangle.is_right_angled", + "pythagoras.calc_hypotenuse", + "pythagoras.calc_side", + }, + ), + ], ), ], ) -def test_target_discovery(sphinx, project, type, kind, expected): - """Ensure that we can correctly discover role targets to suggest.""" +def test_role_target_completions(sphinx, text, setup, caplog): + """Ensure that we can offer correct role target suggestions.""" + + caplog.set_level(logging.DEBUG) + project, expected, unexpected = setup - rst = Mock() + rst = mock.Mock() rst.app = sphinx(project) - rst.app.builder.read() + rst.logger = logging.getLogger("rst") - completion = RoleTargetCompletion(rst) - completion.discover_targets() + feature = Roles(rst) + feature.initialize() - assert type in completion.targets - assert expected == {item.label for item in completion.targets[type]} - assert kind == completion.targets[type][0].kind + completion_test(feature, text, expected, unexpected) From 063098152218cbc2b8626963cca915712aa9ccae Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Fri, 19 Feb 2021 15:25:28 +0000 Subject: [PATCH 03/41] Add domain support to intersphinx completions (#106) Closes #71 --- lib/esbonio/changes/106.feature.rst | 1 + lib/esbonio/esbonio/lsp/__init__.py | 1 + lib/esbonio/esbonio/lsp/directives.py | 2 +- lib/esbonio/esbonio/lsp/intersphinx.py | 227 ++++++++++++++++++ lib/esbonio/esbonio/lsp/roles.py | 177 +------------- lib/esbonio/esbonio/lsp/testing.py | 41 +++- .../tests/data/sphinx-extensions/conf.py | 2 +- .../tests/lsp/completion/test_integration.py | 217 ----------------- lib/esbonio/tests/lsp/test_intersphinx.py | 164 +++++++++++++ lib/esbonio/tests/lsp/test_roles.py | 15 +- 10 files changed, 442 insertions(+), 405 deletions(-) create mode 100644 lib/esbonio/changes/106.feature.rst create mode 100644 lib/esbonio/esbonio/lsp/intersphinx.py delete mode 100644 lib/esbonio/tests/lsp/completion/test_integration.py create mode 100644 lib/esbonio/tests/lsp/test_intersphinx.py diff --git a/lib/esbonio/changes/106.feature.rst b/lib/esbonio/changes/106.feature.rst new file mode 100644 index 00000000..0f6fe105 --- /dev/null +++ b/lib/esbonio/changes/106.feature.rst @@ -0,0 +1 @@ +Intersphinx completions are now domain aware diff --git a/lib/esbonio/esbonio/lsp/__init__.py b/lib/esbonio/esbonio/lsp/__init__.py index 490728e8..8a3f020e 100644 --- a/lib/esbonio/esbonio/lsp/__init__.py +++ b/lib/esbonio/esbonio/lsp/__init__.py @@ -22,6 +22,7 @@ "esbonio.lsp.sphinx", "esbonio.lsp.directives", "esbonio.lsp.roles", + "esbonio.lsp.intersphinx", ] diff --git a/lib/esbonio/esbonio/lsp/directives.py b/lib/esbonio/esbonio/lsp/directives.py index a3b29109..025dd604 100644 --- a/lib/esbonio/esbonio/lsp/directives.py +++ b/lib/esbonio/esbonio/lsp/directives.py @@ -85,7 +85,7 @@ def discover(self): } self.logger.info("Discovered %s directives", len(self.directives)) - self.logger.debug(self.directives.keys()) + self.logger.debug("Directives: %s", list(self.directives.keys())) def resolve_directive(self, directive: Union[Directive, Tuple[str]]): diff --git a/lib/esbonio/esbonio/lsp/intersphinx.py b/lib/esbonio/esbonio/lsp/intersphinx.py new file mode 100644 index 00000000..434d8c10 --- /dev/null +++ b/lib/esbonio/esbonio/lsp/intersphinx.py @@ -0,0 +1,227 @@ +"""Intersphinx support.""" +import re + +from typing import List +from pygls.types import ( + CompletionItem, + CompletionItemKind, + Position, +) +from pygls.workspace import Document + +from esbonio.lsp import RstLanguageServer, LanguageFeature +from esbonio.lsp.roles import PARTIAL_PLAIN_TARGET, PARTIAL_ALIASED_TARGET, TARGET_KINDS +from esbonio.lsp.sphinx import get_domains + + +PARTIAL_INTER_PLAIN_TARGET = re.compile( + r""" + (^|.*[ ]) # roles must be preceeded by a space, or start the line + (?P: # roles start with the ':' character + (?!:) # make sure the next character is not ':' + (?P[\w]+:)? # there may be a domain namespace + (?P[\w-]*) # followed by the role name + :) # the role name ends with a ':' + ` # the target begins with a '`' + (?P[^<:`]*) # match "plain link" targets + : # projects end with a ':' + $ + """, + re.MULTILINE | re.VERBOSE, +) +"""A regular expression that matches a partial "plain" intersphinx target. + +For example:: + + :ref:`python:som + +Used when generating auto complete suggestions. +""" + +PARTIAL_INTER_ALIASED_TARGET = re.compile( + r""" + (^|.*[ ]) # roles must be preceeded by a space, or start the line + (?P: # roles start with the ':' character + (?!:) # make sure the next character is not ':' + (?P[\w]+:)? # there may be a domain namespace + (?P[\w-]*) # followed by the role name + :) # the role name ends with a ':' + ` # the target begins with a '`'` + .*< # the actual target name starts after a '<' + (?P[^`:]*) # match "aliased" targets + : # projects end with a ':' + $ + """, + re.MULTILINE | re.VERBOSE, +) +"""A regular expression that matches a partial "aliased" intersphinx target. + +For example:: + + :ref:`More Info List[CompletionItem]: + + # As a unfortunate consequence of naming choices, we're in the counter intuitive + # situation where + # + # - A match containing a "target" regex group should suggest project names + # - A match containing a "project" regex group should suggest targets + groups = match.groupdict() + + if "target" in groups: + return self.suggest_projects() + + return self.suggest_targets(match) + + def suggest_projects(self) -> List[CompletionItem]: + self.logger.info("Suggesting projects") + + return list(self.projects.values()) + + def suggest_targets(self, match: "re.Match") -> List[CompletionItem]: + # TODO: Detect if we're in an angle bracket e.g. :ref:`More Info ' to the completion item insert text. + self.logger.info("Suggesting targets") + + role = match.group("name") + domain = match.group("domain") or "" + primary_domain = self.rst.app.config.primary_domain or "" + + self.logger.debug("Suggesting targets for '%s%s'", domain, role) + + # Attempt to find the right key.. + for key in [f"{domain}{role}", f"{primary_domain}:{role}", f"std:{role}"]: + target_types = self.target_types.get(key, None) + self.logger.debug("Targets types for '%s': %s", key, target_types) + + if target_types is not None: + break + + if target_types is None: + return [] + + project = self.targets.get(match.group("project"), None) + if project is None: + return [] + + targets = [] + for target_type in target_types: + items = project.get(target_type, {}) + targets += items.values() + + return targets + + def project_to_completion_item(self, project: str) -> CompletionItem: + return CompletionItem( + project, detail="intersphinx", kind=CompletionItemKind.Module + ) + + def target_to_completion_item( + self, label: str, target, target_type: str + ) -> CompletionItem: + kind = TARGET_KINDS.get(target_type, CompletionItemKind.Reference) + source, version, _, display = target + + if display == "-": + display = label + + if version: + version = f" v{version}" + + detail = f"{display} - {source}{version}" + + return CompletionItem(label, kind=kind, detail=detail, insert_text=label) + + +def setup(rst: RstLanguageServer): + intersphinx = InterSphinx(rst) + rst.add_feature(intersphinx) diff --git a/lib/esbonio/esbonio/lsp/roles.py b/lib/esbonio/esbonio/lsp/roles.py index bea888c1..9dd33ef0 100644 --- a/lib/esbonio/esbonio/lsp/roles.py +++ b/lib/esbonio/esbonio/lsp/roles.py @@ -1,7 +1,7 @@ """Role support.""" import re -from typing import Dict, List +from typing import List from docutils.parsers.rst import roles from pygls.types import ( @@ -13,7 +13,6 @@ TextEdit, ) from pygls.workspace import Document -from sphinx.domains import Domain from esbonio.lsp import RstLanguageServer, LanguageFeature, dump from esbonio.lsp.directives import DIRECTIVE @@ -89,14 +88,6 @@ """ -def namespace_to_completion_item(namespace: str) -> CompletionItem: - return CompletionItem( - namespace, - detail="intersphinx namespace", - kind=CompletionItemKind.Module, - ) - - TARGET_KINDS = { "attribute": CompletionItemKind.Field, "doc": CompletionItemKind.File, @@ -117,21 +108,6 @@ def namespace_to_completion_item(namespace: str) -> CompletionItem: } -def intersphinx_target_to_completion_item(label, item, type_) -> CompletionItem: - kind = TARGET_KINDS.get(type_, CompletionItemKind.Reference) - source, version, _, display = item - - if display == "-": - display = label - - if version: - version = f" v{version}" - - detail = f"{display} - {source}{version}" - - return CompletionItem(label, kind=kind, detail=detail, insert_text=label) - - class Roles(LanguageFeature): """Role support for the language server.""" @@ -198,10 +174,10 @@ def discover_roles(self): } self.logger.info("Discovered %s roles", len(self.roles)) - self.logger.info("Discovered %s target types", len(self.target_types)) + self.logger.debug("Roles: %s", list(self.roles.keys())) - self.logger.debug(self.roles.keys()) - self.logger.debug(self.target_types) + self.logger.info("Discovered %s target types", len(self.target_types)) + self.logger.debug("Target types: %s", self.target_types) def discover_targets(self): """Look up all the targets we can offer as autocomplete suggestions. @@ -362,151 +338,6 @@ def target_object_to_completion_item( ) -class RoleTargets: - """Role target support for the language server.""" - - def __init__(self, rst: RstLanguageServer): - self.rst = rst - - def suggest(self, match, doc, position) -> List[CompletionItem]: - # TODO: Detect if we're in an angle bracket e.g. :ref:`More Info <|` in that - # situation, add the closing '>' to the completion item insert text. - - if match is None: - return [] - - rolename = match.group("name") - types = self.target_types.get(rolename, None) - - if types is None: - return [] - - targets = [] - for type_ in types: - targets += self.targets.get(type_, []) - - return targets - - -class InterSphinxNamespaceCompletion: - """Completion handler for intersphinx namespaces.""" - - def __init__(self, rst: RstLanguageServer): - self.rst = rst - self.namespaces = {} - - def initialize(self): - - if self.rst.app and hasattr(self.rst.app.env, "intersphinx_named_inventory"): - inv = self.rst.app.env.intersphinx_named_inventory - self.namespaces = {v: namespace_to_completion_item(v) for v in inv.keys()} - - self.rst.logger.debug( - "Discovered %s intersphinx namespaces", len(self.namespaces) - ) - - # suggest_triggers = RoleTargetCompletion.suggest_triggers - - def suggest(self, match, doc, position) -> List[CompletionItem]: - return list(self.namespaces.values()) - - -def build_target_type_map(domain: Domain) -> Dict[str, List[str]]: - - types = {} - - for name, obj in domain.object_types.items(): - for role in obj.roles: - objs = types.get(role, None) - - if objs is None: - objs = [] - - objs.append(f"{domain.name}:{name}") - types[role] = objs - - return types - - -class InterSphinxTargetCompletion: - """Completion handler for intersphinx targets""" - - def __init__(self, rst: RstLanguageServer): - self.rst = rst - self.targets = {} - self.target_types = {} - - def initialize(self): - - if self.rst.app and hasattr(self.rst.app.env, "intersphinx_named_inventory"): - inv = self.rst.app.env.intersphinx_named_inventory - domains = self.rst.app.env.domains - - for domain in domains.values(): - self.target_types.update(build_target_type_map(domain)) - - for namespace, types in inv.items(): - self.targets[namespace] = { - type_: { - label: intersphinx_target_to_completion_item(label, item, type_) - for label, item in items.items() - } - for type_, items in types.items() - } - - suggest_triggers = [ - re.compile( - r""" - (^|.*[ ]) # roles must be preceeded by a space, or start the line - : # roles start with the ':' character - (?P[\w-]+) # capture the role name, suggestions will change based on it - : # the role name ends with a ':' - ` # the target begins with a '`' - (?P[^<:]*) # match "plain link" targets - : # namespaces end with a ':' - $ - """, - re.MULTILINE | re.VERBOSE, - ), - re.compile( - r""" - (^|.*[ ]) # roles must be preceeded by a space, or start the line - : # roles start with the ':' character - (?P[\w-]+) # capture the role name, suggestions will change based on it - : # the role name ends with a ':' - ` # the target begins with a '`'` - .*< # the actual target name starts after a '<' - (?P[^:]*) # match "aliased" targets - : # namespaces end with a ':' - $ - """, - re.MULTILINE | re.VERBOSE, - ), - ] - - def suggest(self, match, doc, position) -> List[CompletionItem]: - # TODO: Detect if we're in an angle bracket e.g. :ref:`More Info <|` in that - # situation, add the closing '>' to the completion item insert text. - - namespace = match.group("namespace") - rolename = match.group("name") - - types = self.target_types.get(rolename, None) - if types is None: - return [] - - namespace = self.targets.get(namespace, None) - if namespace is None: - return [] - - targets = [] - for type_ in types: - items = namespace.get(type_, {}) - targets += items.values() - - return targets - - def setup(rst: RstLanguageServer): role_completion = Roles(rst) rst.add_feature(role_completion) diff --git a/lib/esbonio/esbonio/lsp/testing.py b/lib/esbonio/esbonio/lsp/testing.py index c0c8a1fa..e2acdffd 100644 --- a/lib/esbonio/esbonio/lsp/testing.py +++ b/lib/esbonio/esbonio/lsp/testing.py @@ -1,7 +1,7 @@ """Utility functions to help with testing Language Server features.""" import logging -from typing import Optional, Set +from typing import List, Optional, Set from pygls.types import Position from pygls.workspace import Document @@ -9,6 +9,45 @@ logger = logging.getLogger(__name__) +def role_target_patterns(name: str) -> List[str]: + """Return a number of example role target patterns. + + These correspond to cases where role target completions may be generated. + + Parameters + ---------- + name: + The name of the role to generate examples for + """ + return [ + s.format(name) + for s in [":{}:`", ":{}:`More Info <", " :{}:`", " :{}:`Some Label <"] + ] + + +def intersphinx_target_patterns(name: str, project: str) -> List[str]: + """Return a number of example intersphinx target patterns. + + These correspond to cases where target completions may be generated + + Parameters + ---------- + name: str + The name of the role to generate examples for + project: str + The name of the project to generate examples for + """ + return [ + s.format(name, project) + for s in [ + ":{}:`{}:", + ":{}:`More Info <{}:", + " :{}:`{}:", + " :{}:`Some Label <{}:", + ] + ] + + def completion_test( feature, text: str, expected: Optional[Set[str]], unexpected: Optional[Set[str]] ): diff --git a/lib/esbonio/tests/data/sphinx-extensions/conf.py b/lib/esbonio/tests/data/sphinx-extensions/conf.py index 6dd79b80..c75d1b61 100644 --- a/lib/esbonio/tests/data/sphinx-extensions/conf.py +++ b/lib/esbonio/tests/data/sphinx-extensions/conf.py @@ -28,7 +28,7 @@ extensions = ["sphinx.ext.doctest", "sphinx.ext.intersphinx"] intersphinx_mapping = { - "python": ("https://docs.python.org/3/", None), + "python": ("https://docs.python.org/3", None), "sphinx": ("https://www.sphinx-doc.org/en/master", None), } diff --git a/lib/esbonio/tests/lsp/completion/test_integration.py b/lib/esbonio/tests/lsp/completion/test_integration.py deleted file mode 100644 index e8c4b18f..00000000 --- a/lib/esbonio/tests/lsp/completion/test_integration.py +++ /dev/null @@ -1,217 +0,0 @@ -import itertools -import pathlib -import time - -from typing import Set - -import py.test - -from pygls.features import ( - COMPLETION, - INITIALIZE, - TEXT_DOCUMENT_DID_CHANGE, - TEXT_DOCUMENT_DID_OPEN, -) -from pygls.server import LanguageServer -from pygls.types import ( - CompletionContext, - CompletionParams, - CompletionTriggerKind, - DidChangeTextDocumentParams, - DidOpenTextDocumentParams, - Position, - Range, - TextDocumentContentChangeEvent, - TextDocumentIdentifier, - TextDocumentItem, - VersionedTextDocumentIdentifier, -) - -WAIT = 0.1 # How long should we sleep after an lsp.notify(...) - - -def do_completion_test( - client: LanguageServer, - server: LanguageServer, - root: pathlib.Path, - filename: str, - text: str, - expected: Set[str], - insert_newline: bool = True, -): - """A generic helper for performing completion tests. - - Being an integration test, it is quite involved as it has to use the protocol to - take the server to a point where it can provide completion suggestions. As part of - the setup, this helper - - - Sends an 'initialize' request to the server, setting the workspace root. - - Sends a 'textDocument/didOpen' notification, loading the specified document in the - server's workspace - - Sends a 'textDocument/didChange' notification, inserting the text we want - suggestions for - - Sends a 'completion' request, and ensures that the expected completed items are in - the response - - Currently this method is only capable of ensuring that item labels are as expected, - none of the other CompletionItem fields are inspected. This method is also not capable - of ensuring particular items are NOT suggested. - - Parameters - ---------- - client: - The client LanguageServer instance - server: - The server LanguageServer instance - root: - The directory to use as the workspace root - filename: - The file to open for the test, relative to the workspace root - text: - The text to insert, this is the text this method requests completions for. - Note this CANNOT contain any newlines. - expected: - The CompletionItem labels that should be returned. This does not have to be - exhaustive. - insert_newline: - Flag to indicate if a newline should be inserted before the given ``text`` - """ - - # Initialize the language server. - response = client.lsp.send_request( - INITIALIZE, {"processId": 1234, "rootUri": root.as_uri(), "capabilities": None} - ).result(timeout=2) - - # Ensure that the server has configured itself correctly. - assert server.workspace.root_uri == root.as_uri() - - # Ensure that server broadcasted the fact that it supports completions. - provider = response.capabilities.completionProvider - assert set(provider.triggerCharacters) == {".", ":", "`", "<"} - - # Let the server know that we have recevied the response. - # client.lsp.notify(INITIALIZED) - # time.sleep(WAIT) - - # Let's open a file to edit. - testfile = root / filename - testuri = testfile.as_uri() - content = testfile.read_text() - - client.lsp.notify( - TEXT_DOCUMENT_DID_OPEN, - DidOpenTextDocumentParams(TextDocumentItem(testuri, "rst", 1, content)), - ) - - time.sleep(WAIT) - assert len(server.lsp.workspace.documents) == 1 - - # With the setup out of the way, let's type the text we want completion suggestions - # for - start = len(content.splitlines()) + insert_newline - text = "\n" + text if insert_newline else text - - client.lsp.notify( - TEXT_DOCUMENT_DID_CHANGE, - DidChangeTextDocumentParams( - VersionedTextDocumentIdentifier(testuri, 2), - [ - TextDocumentContentChangeEvent( - Range(Position(start, 0), Position(start, 0)), text=text - ) - ], - ), - ) - - time.sleep(WAIT) - # Now make the completion request and check to make sure we get the appropriate - # response - response = client.lsp.send_request( - COMPLETION, - CompletionParams( - TextDocumentIdentifier(testuri), - Position(start, len(text) + 1), - CompletionContext(trigger_kind=CompletionTriggerKind.Invoked), - ), - ).result(timeout=2) - - actual = {item.label for item in response.items} - missing = expected - actual - - assert len(missing) == 0, "Missing expected items, {}".format(missing) - - -def intersphinx_patterns(rolename, namespace): - return [ - s.format(rolename, namespace) - for s in [ - ":{}:`{}:", - ":{}:`More Info <{}:", - " :{}:`{}:", - " :{}:`Some Label <{}:", - ] - ] - - -@py.test.mark.skip -@py.test.mark.parametrize( - "text,setup", - [ - *itertools.product( - intersphinx_patterns("ref", "python"), - [ - ("sphinx-default", set()), - ( - "sphinx-extensions", - {"configparser-objects", "types", "whatsnew-index"}, - ), - ], - ), - *itertools.product( - intersphinx_patterns("class", "python"), - [ - ("sphinx-default", set()), - ( - "sphinx-extensions", - {"abc.ABCMeta", "logging.StreamHandler", "zipfile.ZipInfo"}, - ), - ], - ), - *itertools.product( - intersphinx_patterns("ref", "sphinx"), - [ - ("sphinx-default", set()), - ( - "sphinx-extensions", - { - "basic-domain-markup", - "extension-tutorials-index", - "writing-builders", - }, - ), - ], - ), - *itertools.product( - intersphinx_patterns("class", "sphinx"), - [ - ("sphinx-default", set()), - ( - "sphinx-extensions", - { - "sphinx.addnodes.desc", - "sphinx.builders.Builder", - "sphinxcontrib.websupport.WebSupport", - }, - ), - ], - ), - ], -) -def test_expected_completions(client_server, testdata, text, setup): - """Ensure that we can offer the correct completion suggestions.""" - - client, server = client_server - project, expected = setup - root = testdata(project, path_only=True) - - do_completion_test(client, server, root, "index.rst", text, expected) diff --git a/lib/esbonio/tests/lsp/test_intersphinx.py b/lib/esbonio/tests/lsp/test_intersphinx.py new file mode 100644 index 00000000..c8a72a94 --- /dev/null +++ b/lib/esbonio/tests/lsp/test_intersphinx.py @@ -0,0 +1,164 @@ +import itertools +import logging +import unittest.mock as mock + +import py.test + +from esbonio.lsp.intersphinx import InterSphinx +from esbonio.lsp.testing import ( + completion_test, + role_target_patterns, + intersphinx_target_patterns, +) + + +@py.test.fixture(scope="session") +def intersphinx(sphinx): + """Fixture that returns the ``InterSphinx`` feature for a given project. + + Indexing the inventories for every test case adds a noticable overhead + to the test suite and we don't really gain much from it. This caches instances + based on project to speed things up. + """ + + instances = {} + + def cache(project): + + if project in instances: + return instances[project] + + rst = mock.Mock() + rst.app = sphinx(project) + rst.logger = logging.getLogger("rst") + + feature = InterSphinx(rst) + feature.initialize() + instances[project] = feature + + return feature + + return cache + + +@py.test.mark.parametrize( + "text, setup", + [ + # Standard domain + *itertools.product( + role_target_patterns("doc"), + [ + ("sphinx-default", set(), {"python", "sphinx"}), + ("sphinx-extensions", {"python", "sphinx"}, set()), + ], + ), + *itertools.product( + role_target_patterns("ref"), + [ + ("sphinx-default", set(), {"python", "sphinx"}), + ("sphinx-extensions", {"python", "sphinx"}, set()), + ], + ), + *itertools.product( + role_target_patterns("func"), + [ + ("sphinx-default", set(), {"python", "shphinx"}), + ("sphinx-extensions", {"python", "sphinx"}, set()), + ], + ), + ], +) +def test_project_completions(intersphinx, text, setup): + """Ensure that we can offer the correct project completions.""" + + project, expected, unexpected = setup + feature = intersphinx(project) + + completion_test(feature, text, expected, unexpected) + + +@py.test.mark.parametrize( + "text,setup", + [ + *itertools.product( + [ + *intersphinx_target_patterns("ref", "python"), + *intersphinx_target_patterns("std:ref", "python"), + ], + [ + ( + "sphinx-default", + set(), + {"configparser-objects", "types", "whatsnew-index"}, + ), + ( + "sphinx-extensions", + {"configparser-objects", "types", "whatsnew-index"}, + set(), + ), + ], + ), + *itertools.product( + [ + *intersphinx_target_patterns("ref", "sphinx"), + *intersphinx_target_patterns("std:ref", "sphinx"), + ], + [ + ( + "sphinx-default", + set(), + { + "basic-domain-markup", + "extension-tutorials-index", + "writing-builders", + }, + ), + ( + "sphinx-extensions", + { + "basic-domain-markup", + "extension-tutorials-index", + "writing-builders", + }, + set(), + ), + ], + ), + *itertools.product( + intersphinx_target_patterns("func", "python"), + [ + ( + "sphinx-default", + set(), + {"_Py_c_sum", "PyErr_Print", "PyUnicode_Count"}, + ), + ( + "sphinx-extensions", + {"_Py_c_sum", "PyErr_Print", "PyUnicode_Count"}, + set(), + ), + ], + ), + *itertools.product( + intersphinx_target_patterns("py:func", "python"), + [ + ( + "sphinx-default", + set(), + {"abc.abstractmethod", "msvcrt.locking", "types.new_class"}, + ), + ( + "sphinx-extensions", + {"abc.abstractmethod", "msvcrt.locking", "types.new_class"}, + set(), + ), + ], + ), + ], +) +def test_target_completions(intersphinx, text, setup): + """Ensure that we can offer the correct target completions.""" + project, expected, unexpected = setup + feature = intersphinx(project) + + completion_test(feature, text, expected, unexpected) diff --git a/lib/esbonio/tests/lsp/test_roles.py b/lib/esbonio/tests/lsp/test_roles.py index c6118c92..29700561 100644 --- a/lib/esbonio/tests/lsp/test_roles.py +++ b/lib/esbonio/tests/lsp/test_roles.py @@ -1,13 +1,12 @@ +"""Utilities and helpers for writing tests around the Language Server.""" import itertools import logging import unittest.mock as mock import py.test -from pygls.types import CompletionItemKind - from esbonio.lsp.roles import Roles -from esbonio.lsp.testing import completion_test +from esbonio.lsp.testing import completion_test, role_target_patterns C_EXPECTED = {"c:func", "c:macro"} C_UNEXPECTED = {"ref", "doc", "py:func", "py:mod"} @@ -76,13 +75,6 @@ def test_role_completions(sphinx, project, text, expected, unexpected): completion_test(feature, text, expected, unexpected) -def role_target_patterns(name): - return [ - s.format(name) - for s in [":{}:`", ":{}:`More Info <", " :{}:`", " :{}:`Some Label <"] - ] - - @py.test.mark.parametrize( "text,setup", [ @@ -249,10 +241,9 @@ def role_target_patterns(name): ), ], ) -def test_role_target_completions(sphinx, text, setup, caplog): +def test_role_target_completions(sphinx, text, setup): """Ensure that we can offer correct role target suggestions.""" - caplog.set_level(logging.DEBUG) project, expected, unexpected = setup rst = mock.Mock() From a57dc621cd56a66912e28103c078df25d2b0b23a Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Fri, 19 Feb 2021 15:43:31 +0000 Subject: [PATCH 04/41] Attempt to create a workflow that will eventually manage projects --- .github/workflows/automation.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/automation.yml diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml new file mode 100644 index 00000000..5800adcf --- /dev/null +++ b/.github/workflows/automation.yml @@ -0,0 +1,14 @@ +name: Automation + +on: + issues: + types: + - labeled + +jobs: + issues: + runs-on: ubuntu-latest + steps: + - run: | + echo "${{ event.payload.issue }}" + echo "${{ event.payload.label }}" From d59e11e1c4f18df208223514018578ea9292b7bb Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Fri, 19 Feb 2021 15:52:44 +0000 Subject: [PATCH 05/41] Hopefully fix variable reference --- .github/workflows/automation.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index 5800adcf..0ecea40e 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -10,5 +10,6 @@ jobs: runs-on: ubuntu-latest steps: - run: | - echo "${{ event.payload.issue }}" - echo "${{ event.payload.label }}" + echo "${{ github.event.payload.issue }}" + echo "${{ github.event.payload.label }}" + From ea2b665f4a80a3309b722576b6fcb6e149e88390 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Fri, 19 Feb 2021 15:59:50 +0000 Subject: [PATCH 06/41] Trying to figure out how to reference values --- .github/workflows/automation.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index 0ecea40e..a2c7cc9d 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -8,8 +8,15 @@ on: jobs: issues: runs-on: ubuntu-latest + name: Issue Automation steps: - - run: | - echo "${{ github.event.payload.issue }}" - echo "${{ github.event.payload.label }}" + - name: Debug Info + env: + ISSUE: ${{ github.event.payload.issue }} + LABEL: ${{ github.event.payload.label }} + run: | + echo "Event Name: ${{ github.event_name }}" + echo "${ISSUE}" + echo "${LABEL}" + echo "${{ github.event }}" From 38203423551db0d6083038f5c76400323d8fde24 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Fri, 19 Feb 2021 21:38:33 +0000 Subject: [PATCH 07/41] Hopefully getting somewhere --- .github/workflows/automation.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index a2c7cc9d..5a679012 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -11,12 +11,7 @@ jobs: name: Issue Automation steps: - name: Debug Info - env: - ISSUE: ${{ github.event.payload.issue }} - LABEL: ${{ github.event.payload.label }} run: | echo "Event Name: ${{ github.event_name }}" - echo "${ISSUE}" - echo "${LABEL}" - echo "${{ github.event }}" - + echo "${{ toJSON(github.event) }}" + jq -h From 71ad466b018c63d8da119912924c42a0121ed867 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Fri, 19 Feb 2021 21:41:57 +0000 Subject: [PATCH 08/41] Apparently it needs to be an env variable? --- .github/workflows/automation.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index 5a679012..7b1deda5 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -11,7 +11,9 @@ jobs: name: Issue Automation steps: - name: Debug Info + env: + EVENT: ${{ toJSON(github.event) }} run: | echo "Event Name: ${{ github.event_name }}" - echo "${{ toJSON(github.event) }}" + echo "${EVENT}" jq -h From 60a678c897b0e6897c06a8265ba70eb2d18f7811 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 18:23:00 +0000 Subject: [PATCH 09/41] First attempt at adding issues to projects --- .github/workflows/automation.yml | 4 ++- scripts/project-management.sh | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100755 scripts/project-management.sh diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index 7b1deda5..101f1c9b 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -13,7 +13,9 @@ jobs: - name: Debug Info env: EVENT: ${{ toJSON(github.event) }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "Event Name: ${{ github.event_name }}" echo "${EVENT}" - jq -h + + ./scripts/project-management.sh diff --git a/scripts/project-management.sh b/scripts/project-management.sh new file mode 100755 index 00000000..88daddf1 --- /dev/null +++ b/scripts/project-management.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# This script is used to help automate project management in GitHub +# It expects to be called in a GitHub actions workflow with the following setup +# +# EVENT = github.event + +# Configuration +# +# Here follows a bunch of magic numbers which correspond to IDs of the API +# objects we're insterested in. +LSP_PROJECT=11250171 +LSP_BACKLOG=12653773 + + +PREVIEW_HEADER="application/vnd.github.inertia-preview+json" + +add_to_project () { + + issue_id=$1 + label_name=$2 + + column_id="" + + case "${label_name}" in + lsp) + column_id=$LSP_BACKLOG + ;; + *) + echo "Unknown label '${label_name}', doing nothing" + return + ;; + esac + + echo "Adding issue '${issue_id}' to column '${column_id}'" + curl -X POST "https://api.github.com/projects/columns/${column_id}/cards" \ + -H "Accept: ${PREVIEW_HEADER}" \ + -H "Authorization: ${GITHUB_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{\"content_id\": ${issue_id}, \"content_type\": \"Issue\"}" +} + + +action=$(echo "${EVENT}" | jq -r .action) +label_name=$(echo "${EVENT}" | jq -r .label.name) +issue=$(echo "${EVENT}" | jq -r .issue.id ) + +echo "Action: ${action}" +echo "Label: ${label_name}" +echo "Issue Id: ${issue}" + +case "$action" in + labeled) + add_to_project "${issue}" "${label_name}" + ;; + *) + echo "Unknown action '${action}', doing nothing" +esac From 0199e41e452a4bbe84a3c002c7f30f5c95631070 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 18:26:27 +0000 Subject: [PATCH 10/41] Now need to checkout the repo --- .github/workflows/automation.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index 101f1c9b..3e077fe1 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -10,6 +10,8 @@ jobs: runs-on: ubuntu-latest name: Issue Automation steps: + - uses: 'actions/checkout@v2' + - name: Debug Info env: EVENT: ${{ toJSON(github.event) }} From 0c4afc22ba45959b58e1a876b911be6e2abf471e Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 18:31:56 +0000 Subject: [PATCH 11/41] Attempt to debug authentication --- .github/workflows/automation.yml | 4 ++-- scripts/project-management.sh | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index 3e077fe1..4666e568 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -11,13 +11,13 @@ jobs: name: Issue Automation steps: - uses: 'actions/checkout@v2' - + - name: Debug Info env: EVENT: ${{ toJSON(github.event) }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "Event Name: ${{ github.event_name }}" - echo "${EVENT}" + # echo "${EVENT}" ./scripts/project-management.sh diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 88daddf1..a99a50ed 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -15,6 +15,7 @@ LSP_BACKLOG=12653773 PREVIEW_HEADER="application/vnd.github.inertia-preview+json" + add_to_project () { issue_id=$1 @@ -33,14 +34,24 @@ add_to_project () { esac echo "Adding issue '${issue_id}' to column '${column_id}'" - curl -X POST "https://api.github.com/projects/columns/${column_id}/cards" \ + curl -s -X POST "https://api.github.com/projects/columns/${column_id}/cards" \ -H "Accept: ${PREVIEW_HEADER}" \ - -H "Authorization: ${GITHUB_TOKEN}" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ -H "Content-Type: application/json" \ -d "{\"content_id\": ${issue_id}, \"content_type\": \"Issue\"}" } +# +# Script start. +# + + +if [ -z "${GITHUB_TOKEN}" ] ; + echo "Github token is not set." + exit 1 +fi + action=$(echo "${EVENT}" | jq -r .action) label_name=$(echo "${EVENT}" | jq -r .label.name) issue=$(echo "${EVENT}" | jq -r .issue.id ) From 50225c5a98f948e75aaef8c9991d1a96fb4f05dd Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 18:35:23 +0000 Subject: [PATCH 12/41] Fix syntax --- scripts/project-management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index a99a50ed..be2c2c8f 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -47,7 +47,7 @@ add_to_project () { # -if [ -z "${GITHUB_TOKEN}" ] ; +if [ -z "${GITHUB_TOKEN}" ]; then echo "Github token is not set." exit 1 fi From 4b49b94eff5751fa53d698c43dee9519de32cdd7 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 18:40:26 +0000 Subject: [PATCH 13/41] Add vscode project ids --- scripts/project-management.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index be2c2c8f..813cb2ee 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -4,6 +4,7 @@ # It expects to be called in a GitHub actions workflow with the following setup # # EVENT = github.event +# GITHUB_TOKEN = secrets.GITHUB_TOKEN # Configuration # @@ -12,6 +13,8 @@ LSP_PROJECT=11250171 LSP_BACKLOG=12653773 +VSCODE_PROJECT=11250281 +VSCODE_BACKLOG=12653879 PREVIEW_HEADER="application/vnd.github.inertia-preview+json" @@ -27,6 +30,8 @@ add_to_project () { lsp) column_id=$LSP_BACKLOG ;; + vscode) + column_id=$VSCODE_BACKLOG *) echo "Unknown label '${label_name}', doing nothing" return From 96b93035ff883c45721363f1c3f0fab0016d7774 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 18:42:36 +0000 Subject: [PATCH 14/41] Fix syntax --- scripts/project-management.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 813cb2ee..6fe61e98 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -32,6 +32,7 @@ add_to_project () { ;; vscode) column_id=$VSCODE_BACKLOG + ;; *) echo "Unknown label '${label_name}', doing nothing" return From fd98fabfc7acbe40944846a34c994fb46058d692 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 18:48:02 +0000 Subject: [PATCH 15/41] Listen for unlabeled events --- .github/workflows/automation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index 4666e568..711bc0e9 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -4,6 +4,7 @@ on: issues: types: - labeled + - unlabeled jobs: issues: From b73f1f141bbf237bf9d2d0668910dd9e4584fe34 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 19:16:13 +0000 Subject: [PATCH 16/41] Initial attempt at removing issues from projects --- scripts/project-management.sh | 49 ++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 6fe61e98..e3c2b3cf 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -48,6 +48,42 @@ add_to_project () { } +remove_from_project () { + + issue_number=$1 + label_name=$2 + + case "${label_name}" in + lsp) + column_id=$LSP_BACKLOG + ;; + vscode) + column_id=$VSCODE_BACKLOG + ;; + *) + echo "Unknown label '${label_name}', doing nothing" + return + ;; + esac + + # Need to look to see which card corresponds to the issue. + echo "Looking for issue in column '${column_id}'" + card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ + -H "Accept: ${PREVIEW_HEADER}" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r ".[] | select(.content_url | test(\".*/${issue_number}\") | .id") + + if [ -z "${card_id}" ]; then + echo "Couldn't find card for issue '${issue_number}', doing nothing" + return + fi + + echo "Removing card '${card_id}' from column '${column_id}'" + curl -s -X DELETE "https://api.github.com/projects/columns/cards/${card_id}" \ + -H "Accept: ${PREVIEW_HEADER}" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" +} + + # # Script start. # @@ -61,15 +97,20 @@ fi action=$(echo "${EVENT}" | jq -r .action) label_name=$(echo "${EVENT}" | jq -r .label.name) issue=$(echo "${EVENT}" | jq -r .issue.id ) +issue_number=$(echo "${EVENT}" | jq -r .issue.number) -echo "Action: ${action}" -echo "Label: ${label_name}" -echo "Issue Id: ${issue}" +echo "Action: ${action}" +echo "Label: ${label_name}" +echo "Issue Id: ${issue}" +echo "Issue Number: ${issue_number}" case "$action" in labeled) add_to_project "${issue}" "${label_name}" - ;; + ;; + unlabeled) + remove_from_project "${issue_number}" "${label_name}" + ;; *) echo "Unknown action '${action}', doing nothing" esac From 70e1f7ae9817cd403693dd0c6587ddfc22c4f17d Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 19:19:31 +0000 Subject: [PATCH 17/41] Try and fix quotes --- scripts/project-management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index e3c2b3cf..baf13285 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -70,7 +70,7 @@ remove_from_project () { echo "Looking for issue in column '${column_id}'" card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ -H "Accept: ${PREVIEW_HEADER}" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r ".[] | select(.content_url | test(\".*/${issue_number}\") | .id") + -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r "'.[] | select(.content_url | test(\".*/${issue_number}\") | .id'") if [ -z "${card_id}" ]; then echo "Couldn't find card for issue '${issue_number}', doing nothing" From 7b0b791d9975c9652c4914dd2517478a69954b35 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 19:26:50 +0000 Subject: [PATCH 18/41] Attempt to pass args the correct way --- scripts/project-management.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index baf13285..55d91327 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -70,7 +70,7 @@ remove_from_project () { echo "Looking for issue in column '${column_id}'" card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ -H "Accept: ${PREVIEW_HEADER}" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r "'.[] | select(.content_url | test(\".*/${issue_number}\") | .id'") + -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r --arg issue "${issue_number}"'.[] | select(.content_url | test(\".*/$issue\") | .id') if [ -z "${card_id}" ]; then echo "Couldn't find card for issue '${issue_number}', doing nothing" @@ -99,10 +99,12 @@ label_name=$(echo "${EVENT}" | jq -r .label.name) issue=$(echo "${EVENT}" | jq -r .issue.id ) issue_number=$(echo "${EVENT}" | jq -r .issue.number) +echo echo "Action: ${action}" echo "Label: ${label_name}" echo "Issue Id: ${issue}" echo "Issue Number: ${issue_number}" +echo case "$action" in labeled) From 10fa888935b5568a31e0bf472cd0a42306311371 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 19:29:18 +0000 Subject: [PATCH 19/41] Fix arg order --- scripts/project-management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 55d91327..fd5ed12f 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -70,7 +70,7 @@ remove_from_project () { echo "Looking for issue in column '${column_id}'" card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ -H "Accept: ${PREVIEW_HEADER}" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r --arg issue "${issue_number}"'.[] | select(.content_url | test(\".*/$issue\") | .id') + -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(\".*/$issue\") | .id') if [ -z "${card_id}" ]; then echo "Couldn't find card for issue '${issue_number}', doing nothing" From bc6610cd636e0861f987040b54354cfb52e4cb6e Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 19:32:33 +0000 Subject: [PATCH 20/41] Probably don't need to escape the quotes now --- scripts/project-management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index fd5ed12f..53644679 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -70,7 +70,7 @@ remove_from_project () { echo "Looking for issue in column '${column_id}'" card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ -H "Accept: ${PREVIEW_HEADER}" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(\".*/$issue\") | .id') + -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/$issue") | .id') if [ -z "${card_id}" ]; then echo "Couldn't find card for issue '${issue_number}', doing nothing" From b3a17cb735beb125d04edb6d82b531bbbd9a686a Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 19:34:59 +0000 Subject: [PATCH 21/41] Fix missing parenthesis --- scripts/project-management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 53644679..47ab24fc 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -70,7 +70,7 @@ remove_from_project () { echo "Looking for issue in column '${column_id}'" card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ -H "Accept: ${PREVIEW_HEADER}" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/$issue") | .id') + -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/$issue")) | .id') if [ -z "${card_id}" ]; then echo "Couldn't find card for issue '${issue_number}', doing nothing" From 413d82c4243edefdfe23ddeb0ff914a122f97c47 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 19:37:53 +0000 Subject: [PATCH 22/41] Debugging --- scripts/project-management.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 47ab24fc..2c57634a 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -68,6 +68,14 @@ remove_from_project () { # Need to look to see which card corresponds to the issue. echo "Looking for issue in column '${column_id}'" + curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ + -H "Accept: ${PREVIEW_HEADER}" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" + + curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ + -H "Accept: ${PREVIEW_HEADER}" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/$issue")) | .id' + card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ -H "Accept: ${PREVIEW_HEADER}" \ -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/$issue")) | .id') From 20d9a7c5b41007af5d6cbfc77636807244e41c56 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 19:44:08 +0000 Subject: [PATCH 23/41] Attempt to fix pattern --- scripts/project-management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 2c57634a..7399947f 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -74,7 +74,7 @@ remove_from_project () { curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ -H "Accept: ${PREVIEW_HEADER}" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/$issue")) | .id' + -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/" + $issue)) | .id' card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ -H "Accept: ${PREVIEW_HEADER}" \ From 85648e51fc97d68c00bfadaeb9c46d1e008dd924 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 19:48:13 +0000 Subject: [PATCH 24/41] In theory, this should now work --- scripts/project-management.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 7399947f..f5dd0fc9 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -68,17 +68,9 @@ remove_from_project () { # Need to look to see which card corresponds to the issue. echo "Looking for issue in column '${column_id}'" - curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ - -H "Accept: ${PREVIEW_HEADER}" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" - - curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ - -H "Accept: ${PREVIEW_HEADER}" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/" + $issue)) | .id' - card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ -H "Accept: ${PREVIEW_HEADER}" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/$issue")) | .id') + -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/" + $issue)) | .id') if [ -z "${card_id}" ]; then echo "Couldn't find card for issue '${issue_number}', doing nothing" From 878cab0fb2712518ea918832e6e08fd1abe92748 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 20:09:31 +0000 Subject: [PATCH 25/41] First attempt at assigned automation --- .github/workflows/automation.yml | 1 + scripts/project-management.sh | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index 711bc0e9..327abc54 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -3,6 +3,7 @@ name: Automation on: issues: types: + - assigned - labeled - unlabeled diff --git a/scripts/project-management.sh b/scripts/project-management.sh index f5dd0fc9..d13ddf35 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -12,9 +12,11 @@ # objects we're insterested in. LSP_PROJECT=11250171 LSP_BACKLOG=12653773 +LSP_PROGRESS=12653764 VSCODE_PROJECT=11250281 VSCODE_BACKLOG=12653879 +VSCODE_PROGRESS=12653872 PREVIEW_HEADER="application/vnd.github.inertia-preview+json" @@ -84,6 +86,44 @@ remove_from_project () { } +card_in_progress () { + + issue_number=$1 + label_name=$2 + + case "${label_name}" in + lsp) + column_id=$LSP_PROGRESS + ;; + vscode) + column_id=$VSCODE_PROGRESS + ;; + *) + echo "Unknown label '${label_name}', doing nothing" + return + ;; + esac + + # Need to look to see which card corresponds to the issue + echo "Looking for issue in column '${column_id}'" + card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ + -H "Accept: ${PREVIEW_HEADER}" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/" + $issue)) | .id') + + if [ -z "${card_id}" ]; then + echo "Couldn't find card for issue '${issue_number}', doing nothing" + return + fi + + echo "Moving card '${card_id}' to column '${column_id}'" + curl -s -X POST 'https://api.github.com/projects/columns/cards/${card_id}/moves' \ + -H 'Accept: ${PREVIEW_HEADER}' \ + -H 'Authorization: Bearer ${GITHUB_TOKEN}' \ + -H 'Content-Type: application/json' \ + -d '{"column_id": ${column_id}, "position": "top"}' +} + + # # Script start. # @@ -107,6 +147,9 @@ echo "Issue Number: ${issue_number}" echo case "$action" in + assigned) + card_in_progress "${issue_number}" "${label_name}" + ;; labeled) add_to_project "${issue}" "${label_name}" ;; From 7ba83f6d372f9403b760499477faaa60054b95ba Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 20:16:41 +0000 Subject: [PATCH 26/41] Attempt to find the project label --- scripts/project-management.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index d13ddf35..d33c4d1c 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -148,6 +148,11 @@ echo case "$action" in assigned) + echo + echo "Looking for project label" + label_name=$(echo "${EVENT}" | jq -r '.issue.labels[].name' | grep -E "lsp|vscode") + echo "Label Name: ${label_name}" + card_in_progress "${issue_number}" "${label_name}" ;; labeled) From faa4837c27b7c9096a06f801deb1afd2cc0621b8 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 20:31:57 +0000 Subject: [PATCH 27/41] You have to look for the card in the old column - duh! --- scripts/project-management.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index d33c4d1c..28a72ab5 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -93,10 +93,12 @@ card_in_progress () { case "${label_name}" in lsp) - column_id=$LSP_PROGRESS + new_column_id=$LSP_PROGRESS + old_column_id=$LSP_BACKLOG ;; vscode) - column_id=$VSCODE_PROGRESS + new_column_id=$VSCODE_PROGRESS + old_column_id=$VSCODE_BACKLOG ;; *) echo "Unknown label '${label_name}', doing nothing" @@ -106,7 +108,7 @@ card_in_progress () { # Need to look to see which card corresponds to the issue echo "Looking for issue in column '${column_id}'" - card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${column_id}/cards" \ + card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${old_column_id}/cards" \ -H "Accept: ${PREVIEW_HEADER}" \ -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/" + $issue)) | .id') @@ -120,7 +122,7 @@ card_in_progress () { -H 'Accept: ${PREVIEW_HEADER}' \ -H 'Authorization: Bearer ${GITHUB_TOKEN}' \ -H 'Content-Type: application/json' \ - -d '{"column_id": ${column_id}, "position": "top"}' + -d '{"column_id": ${new_column_id}, "position": "top"}' } From 7d8bcd0d23b66b07106c779b29d7478e683e03fa Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 22:54:55 +0000 Subject: [PATCH 28/41] Fix variable references --- scripts/project-management.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 28a72ab5..9396c5df 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -107,7 +107,7 @@ card_in_progress () { esac # Need to look to see which card corresponds to the issue - echo "Looking for issue in column '${column_id}'" + echo "Looking for issue in column '${old_column_id}'" card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${old_column_id}/cards" \ -H "Accept: ${PREVIEW_HEADER}" \ -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/" + $issue)) | .id') @@ -117,7 +117,7 @@ card_in_progress () { return fi - echo "Moving card '${card_id}' to column '${column_id}'" + echo "Moving card '${card_id}' to column '${new_column_id}'" curl -s -X POST 'https://api.github.com/projects/columns/cards/${card_id}/moves' \ -H 'Accept: ${PREVIEW_HEADER}' \ -H 'Authorization: Bearer ${GITHUB_TOKEN}' \ From a4824fb6a7b8d8c30a847af15d1c8fe8ab22a0c9 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 23:02:48 +0000 Subject: [PATCH 29/41] Trying a different auth token --- .github/workflows/automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index 327abc54..bc56b10a 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -17,7 +17,7 @@ jobs: - name: Debug Info env: EVENT: ${{ toJSON(github.event) }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.API_TOKEN }} run: | echo "Event Name: ${{ github.event_name }}" # echo "${EVENT}" From bd602411c5151d8688fb7bbad8c08e66d820de0c Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 23:07:14 +0000 Subject: [PATCH 30/41] It didn't help --- .github/workflows/automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index bc56b10a..327abc54 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -17,7 +17,7 @@ jobs: - name: Debug Info env: EVENT: ${{ toJSON(github.event) }} - GITHUB_TOKEN: ${{ secrets.API_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "Event Name: ${{ github.event_name }}" # echo "${EVENT}" From 3a6866ee8e1415b84060df818ea3f255327f4f7e Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 23:14:20 +0000 Subject: [PATCH 31/41] It was the wrong type of quotes! --- scripts/project-management.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 9396c5df..1ea86271 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -119,8 +119,8 @@ card_in_progress () { echo "Moving card '${card_id}' to column '${new_column_id}'" curl -s -X POST 'https://api.github.com/projects/columns/cards/${card_id}/moves' \ - -H 'Accept: ${PREVIEW_HEADER}' \ - -H 'Authorization: Bearer ${GITHUB_TOKEN}' \ + -H "Accept: ${PREVIEW_HEADER}" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ -H 'Content-Type: application/json' \ -d '{"column_id": ${new_column_id}, "position": "top"}' } From b67d4b102eee175dc4a7f1d682b23a39750ce159 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 23:17:35 +0000 Subject: [PATCH 32/41] More quotes! --- scripts/project-management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 1ea86271..41cf468a 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -118,7 +118,7 @@ card_in_progress () { fi echo "Moving card '${card_id}' to column '${new_column_id}'" - curl -s -X POST 'https://api.github.com/projects/columns/cards/${card_id}/moves' \ + curl -s -X POST "https://api.github.com/projects/columns/cards/${card_id}/moves" \ -H "Accept: ${PREVIEW_HEADER}" \ -H "Authorization: Bearer ${GITHUB_TOKEN}" \ -H 'Content-Type: application/json' \ From 6b7b324727b1290f330b9a8bf2c10f5dc9606c26 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sat, 20 Feb 2021 23:19:37 +0000 Subject: [PATCH 33/41] Yet more quoting woes --- scripts/project-management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 41cf468a..240ef8ce 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -122,7 +122,7 @@ card_in_progress () { -H "Accept: ${PREVIEW_HEADER}" \ -H "Authorization: Bearer ${GITHUB_TOKEN}" \ -H 'Content-Type: application/json' \ - -d '{"column_id": ${new_column_id}, "position": "top"}' + -d "{\"column_id\": ${new_column_id}, \"position\": \"top\"}" } From f87d3c30ff46286c75f7940c8193b80357a517f6 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sun, 21 Feb 2021 14:17:02 +0000 Subject: [PATCH 34/41] Check for tasks in both todo and backlog columns --- scripts/project-management.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 240ef8ce..9c80d91c 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -12,10 +12,12 @@ # objects we're insterested in. LSP_PROJECT=11250171 LSP_BACKLOG=12653773 +LSP_TODO=12653763 LSP_PROGRESS=12653764 VSCODE_PROJECT=11250281 VSCODE_BACKLOG=12653879 +VSCODE_TODO=12653871 VSCODE_PROGRESS=12653872 PREVIEW_HEADER="application/vnd.github.inertia-preview+json" @@ -94,11 +96,11 @@ card_in_progress () { case "${label_name}" in lsp) new_column_id=$LSP_PROGRESS - old_column_id=$LSP_BACKLOG + old_column_ids=($LSP_BACKLOG $LSP_TODO) ;; vscode) new_column_id=$VSCODE_PROGRESS - old_column_id=$VSCODE_BACKLOG + old_column_ids=($VSCODE_BACKLOG $VSCODE_TODO) ;; *) echo "Unknown label '${label_name}', doing nothing" @@ -107,10 +109,17 @@ card_in_progress () { esac # Need to look to see which card corresponds to the issue - echo "Looking for issue in column '${old_column_id}'" - card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${old_column_id}/cards" \ - -H "Accept: ${PREVIEW_HEADER}" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/" + $issue)) | .id') + for old_column_id in ${old_column_ids[@]}; do + echo "Looking for issue in column '${old_column_id}'" + card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${old_column_id}/cards" \ + -H "Accept: ${PREVIEW_HEADER}" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/" + $issue)) | .id') + + if [ -n "${card_id}" ]; then + echo "Found card '${card_id}' in column '${old_column_id}'" + break + fi + done if [ -z "${card_id}" ]; then echo "Couldn't find card for issue '${issue_number}', doing nothing" From 350b8c428be8bd392f40bd67489e79fe9caa48e4 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Sun, 21 Feb 2021 14:27:09 +0000 Subject: [PATCH 35/41] Add unassignment automation --- .github/workflows/automation.yml | 3 +- scripts/project-management.sh | 47 ++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index 327abc54..612e3f70 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -5,6 +5,7 @@ on: types: - assigned - labeled + - unassigned - unlabeled jobs: @@ -14,7 +15,7 @@ jobs: steps: - uses: 'actions/checkout@v2' - - name: Debug Info + - name: Run Script env: EVENT: ${{ toJSON(github.event) }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/project-management.sh b/scripts/project-management.sh index 9c80d91c..dc3483d2 100755 --- a/scripts/project-management.sh +++ b/scripts/project-management.sh @@ -135,6 +135,45 @@ card_in_progress () { } +card_to_backlog () { + + issue_number=$1 + label_name=$2 + + case "${label_name}" in + lsp) + new_column_id=$LSP_BACKLOG + old_column_id=$LSP_PROGRESS + ;; + vscode) + new_column_id=$VSCODE_BACKLOG + old_column_id=$VSCODE_PROGRESS + ;; + *) + echo "Unknown label '${label_name}', doing nothing" + return + ;; + esac + + # Need to look to see which card corresponds to the issue + echo "Looking for issue in column '${old_column_id}'" + card_id=$(curl -s -X GET "https://api.github.com/projects/columns/${old_column_id}/cards" \ + -H "Accept: ${PREVIEW_HEADER}" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq --arg issue "${issue_number}" -r '.[] | select(.content_url | test(".*/" + $issue)) | .id') + + if [ -z "${card_id}" ]; then + echo "Couldn't find card for issue '${issue_number}', doing nothing" + return + fi + + echo "Moving card '${card_id}' to column '${new_column_id}'" + curl -s -X POST "https://api.github.com/projects/columns/cards/${card_id}/moves" \ + -H "Accept: ${PREVIEW_HEADER}" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "{\"column_id\": ${new_column_id}, \"position\": \"top\"}" +} + # # Script start. # @@ -169,6 +208,14 @@ case "$action" in labeled) add_to_project "${issue}" "${label_name}" ;; + unassigned) + echo + echo "Looking for project label" + label_name=$(echo "${EVENT}" | jq -r '.issue.labels[].name' | grep -E "lsp|vscode") + echo "Label Name: ${label_name}" + + card_to_backlog "${issue_number}" "${label_name}" + ;; unlabeled) remove_from_project "${issue_number}" "${label_name}" ;; From c7563e3d2428bff82c688d44e642b27a473c3997 Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Mon, 22 Feb 2021 20:23:17 +0000 Subject: [PATCH 36/41] Fix doc completion suggestions (#108) --- lib/esbonio/changes/102.fix.rst | 1 + lib/esbonio/esbonio/lsp/intersphinx.py | 19 ++++++++++-- lib/esbonio/esbonio/lsp/roles.py | 42 ++++++++++++++------------ lib/esbonio/tests/lsp/test_roles.py | 19 ++++++++++++ 4 files changed, 59 insertions(+), 22 deletions(-) create mode 100644 lib/esbonio/changes/102.fix.rst diff --git a/lib/esbonio/changes/102.fix.rst b/lib/esbonio/changes/102.fix.rst new file mode 100644 index 00000000..6d22b2f2 --- /dev/null +++ b/lib/esbonio/changes/102.fix.rst @@ -0,0 +1 @@ +Ensure ``:doc:`` completions are specified relative to the project root. \ No newline at end of file diff --git a/lib/esbonio/esbonio/lsp/intersphinx.py b/lib/esbonio/esbonio/lsp/intersphinx.py index 434d8c10..a86c021e 100644 --- a/lib/esbonio/esbonio/lsp/intersphinx.py +++ b/lib/esbonio/esbonio/lsp/intersphinx.py @@ -10,7 +10,12 @@ from pygls.workspace import Document from esbonio.lsp import RstLanguageServer, LanguageFeature -from esbonio.lsp.roles import PARTIAL_PLAIN_TARGET, PARTIAL_ALIASED_TARGET, TARGET_KINDS +from esbonio.lsp.roles import ( + COMPLETION_TARGETS, + DEFAULT_TARGET, + PARTIAL_PLAIN_TARGET, + PARTIAL_ALIASED_TARGET, +) from esbonio.lsp.sphinx import get_domains @@ -208,7 +213,13 @@ def project_to_completion_item(self, project: str) -> CompletionItem: def target_to_completion_item( self, label: str, target, target_type: str ) -> CompletionItem: - kind = TARGET_KINDS.get(target_type, CompletionItemKind.Reference) + + key = target_type + + if ":" in key: + key = ":".join(key.split(":")[1:]) + + completion_type = COMPLETION_TARGETS.get(key, DEFAULT_TARGET) source, version, _, display = target if display == "-": @@ -219,7 +230,9 @@ def target_to_completion_item( detail = f"{display} - {source}{version}" - return CompletionItem(label, kind=kind, detail=detail, insert_text=label) + return CompletionItem( + label, kind=completion_type.kind, detail=detail, insert_text=label + ) def setup(rst: RstLanguageServer): diff --git a/lib/esbonio/esbonio/lsp/roles.py b/lib/esbonio/esbonio/lsp/roles.py index 9dd33ef0..389340d4 100644 --- a/lib/esbonio/esbonio/lsp/roles.py +++ b/lib/esbonio/esbonio/lsp/roles.py @@ -1,4 +1,5 @@ """Role support.""" +import collections import re from typing import List @@ -88,23 +89,18 @@ """ -TARGET_KINDS = { - "attribute": CompletionItemKind.Field, - "doc": CompletionItemKind.File, - "class": CompletionItemKind.Class, - "envvar": CompletionItemKind.Variable, - "function": CompletionItemKind.Function, - "method": CompletionItemKind.Method, - "module": CompletionItemKind.Module, - "py:attribute": CompletionItemKind.Field, - "py:class": CompletionItemKind.Class, - "py:function": CompletionItemKind.Function, - "py:method": CompletionItemKind.Method, - "py:module": CompletionItemKind.Module, - "std:doc": CompletionItemKind.File, - "std:envvar": CompletionItemKind.Variable, - "std:term": CompletionItemKind.Text, - "term": CompletionItemKind.Text, +CompletionTarget = collections.namedtuple("CompletionTarget", "kind,insert_fmt") + +DEFAULT_TARGET = CompletionTarget(CompletionItemKind.Reference, "{name}") +COMPLETION_TARGETS = { + "attribute": CompletionTarget(CompletionItemKind.Field, "{name}"), + "doc": CompletionTarget(CompletionItemKind.File, "/{name}"), + "class": CompletionTarget(CompletionItemKind.Class, "{name}"), + "envvar": CompletionTarget(CompletionItemKind.Variable, "{name}"), + "function": CompletionTarget(CompletionItemKind.Function, "{name}"), + "method": CompletionTarget(CompletionItemKind.Method, "{name}"), + "module": CompletionTarget(CompletionItemKind.Module, "{name}"), + "term": CompletionTarget(CompletionItemKind.Text, "{name}"), } @@ -331,10 +327,18 @@ def target_object_to_completion_item( ) -> CompletionItem: """Convert a target object to its CompletionItem representation.""" - kind = TARGET_KINDS.get(obj_type, CompletionItemKind.Reference) + key = obj_type + + if ":" in key: + _, key = key.split(":") + + target_type = COMPLETION_TARGETS.get(key, DEFAULT_TARGET) return CompletionItem( - name, kind=kind, detail=str(display_name), insert_text=name + name, + kind=target_type.kind, + detail=str(display_name), + insert_text=target_type.insert_fmt.format(name=name), ) diff --git a/lib/esbonio/tests/lsp/test_roles.py b/lib/esbonio/tests/lsp/test_roles.py index 29700561..94769355 100644 --- a/lib/esbonio/tests/lsp/test_roles.py +++ b/lib/esbonio/tests/lsp/test_roles.py @@ -5,6 +5,8 @@ import py.test +from pygls.types import CompletionItemKind + from esbonio.lsp.roles import Roles from esbonio.lsp.testing import completion_test, role_target_patterns @@ -254,3 +256,20 @@ def test_role_target_completions(sphinx, text, setup): feature.initialize() completion_test(feature, text, expected, unexpected) + + +@py.test.mark.parametrize("obj_type", ["doc", "std:doc"]) +def test_doc_target_completion_items(sphinx, obj_type): + """Ensure that we represent ``:doc:`` completion items correctly.""" + + rst = mock.Mock() + rst.app = sphinx("sphinx-default") + rst.logger = logging.getLogger("rst") + + roles = Roles(rst) + item = roles.target_object_to_completion_item("index/example", "Example", obj_type) + + assert item.label == "index/example" + assert item.kind == CompletionItemKind.File + assert item.detail == "Example" + assert item.insertText == "/index/example" From 539430d9b3e1f98f62742189e14cb54dc1e1397e Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Tue, 23 Feb 2021 16:20:00 +0000 Subject: [PATCH 37/41] Ensure conf.py changes trigger a reload of the language server (#109) Closes #83 --- lib/esbonio/changes/83.fix.rst | 1 + lib/esbonio/esbonio/lsp/__init__.py | 29 +++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 lib/esbonio/changes/83.fix.rst diff --git a/lib/esbonio/changes/83.fix.rst b/lib/esbonio/changes/83.fix.rst new file mode 100644 index 00000000..857da829 --- /dev/null +++ b/lib/esbonio/changes/83.fix.rst @@ -0,0 +1 @@ +The language server now reloads when the project's ``conf.py`` is modified diff --git a/lib/esbonio/esbonio/lsp/__init__.py b/lib/esbonio/esbonio/lsp/__init__.py index 8a3f020e..28a74c4e 100644 --- a/lib/esbonio/esbonio/lsp/__init__.py +++ b/lib/esbonio/esbonio/lsp/__init__.py @@ -2,8 +2,10 @@ import importlib import json import logging +import pathlib from typing import List, Optional +from urllib.parse import urlparse, unquote from pygls.features import COMPLETION, INITIALIZE, INITIALIZED, TEXT_DOCUMENT_DID_SAVE from pygls.server import LanguageServer @@ -88,6 +90,13 @@ def load_module(self, mod: str): module.setup(self) + def run_hooks(self, kind: str, *args): + """Run each hook registered of the given kind.""" + + hooks = getattr(self, f"on_{kind}_hooks") + for hook in hooks: + hook(*args) + def get_line_til_position(doc: Document, position: Position) -> str: """Return the line up until the position of the cursor.""" @@ -128,16 +137,12 @@ def create_language_server(modules: List[str]) -> RstLanguageServer: @server.feature(INITIALIZE) def on_initialize(rst: RstLanguageServer, params: InitializeParams): - for init_hook in rst.on_init_hooks: - init_hook() - + rst.run_hooks("init") rst.logger.info("LSP Server Initialized") @server.feature(INITIALIZED) def on_initialized(rst: RstLanguageServer, params): - - for initialized_hook in rst.on_initialized_hooks: - initialized_hook() + rst.run_hooks("initialized") @server.feature(COMPLETION, trigger_characters=[".", ":", "`", "<"]) def on_completion(rst: RstLanguageServer, params: CompletionParams): @@ -160,8 +165,16 @@ def on_completion(rst: RstLanguageServer, params: CompletionParams): @server.feature(TEXT_DOCUMENT_DID_SAVE) def on_save(rst: RstLanguageServer, params: DidSaveTextDocumentParams): + rst.logger.debug("DidSave: %s", params) + + uri = urlparse(params.textDocument.uri) + filepath = pathlib.Path(unquote(uri.path)) + conf_py = pathlib.Path(rst.app.confdir, "conf.py") - for on_save_hook in rst.on_save_hooks: - on_save_hook(params) + # Re-initialize everything if the app's config has changed. + if filepath == conf_py: + rst.run_hooks("init") + else: + rst.run_hooks("save", params) return server From 2299388d6ee270c7b9f820a1575179c2fba3046e Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Tue, 23 Feb 2021 16:32:59 +0000 Subject: [PATCH 38/41] Fix JsonRpcMethodNotFound exceptions when running in VSCode (#110) Closes #91 --- lib/esbonio/changes/91.fix.rst | 2 ++ lib/esbonio/esbonio/lsp/__init__.py | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 lib/esbonio/changes/91.fix.rst diff --git a/lib/esbonio/changes/91.fix.rst b/lib/esbonio/changes/91.fix.rst new file mode 100644 index 00000000..d4c5e496 --- /dev/null +++ b/lib/esbonio/changes/91.fix.rst @@ -0,0 +1,2 @@ +``$/setTraceNotification`` notifications from VSCode no longer cause exceptions to be thrown +in the Language Server. diff --git a/lib/esbonio/esbonio/lsp/__init__.py b/lib/esbonio/esbonio/lsp/__init__.py index 28a74c4e..d24617d1 100644 --- a/lib/esbonio/esbonio/lsp/__init__.py +++ b/lib/esbonio/esbonio/lsp/__init__.py @@ -177,4 +177,9 @@ def on_save(rst: RstLanguageServer, params: DidSaveTextDocumentParams): else: rst.run_hooks("save", params) + @server.feature("$/setTraceNotification") + def vscode_set_trace(rst: RstLanguageServer, params): + """Dummy implementation, stops JsonRpcMethodNotFound exceptions.""" + rst.logger.debug("VSCode set trace: %s", params) + return server From f3e5114656260ef4e7ac3645c60f18e42280dd8c Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Tue, 23 Feb 2021 18:05:04 +0000 Subject: [PATCH 39/41] Lookup `autoxxxx` directive options (#111) Closes #100 --- lib/esbonio/changes/100.bug.rst | 1 + lib/esbonio/esbonio/lsp/directives.py | 27 ++++++++++++- .../tests/data/sphinx-extensions/conf.py | 2 +- lib/esbonio/tests/lsp/test_directives.py | 39 ++++++++++++++++++- 4 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 lib/esbonio/changes/100.bug.rst diff --git a/lib/esbonio/changes/100.bug.rst b/lib/esbonio/changes/100.bug.rst new file mode 100644 index 00000000..3694ba34 --- /dev/null +++ b/lib/esbonio/changes/100.bug.rst @@ -0,0 +1 @@ +The Language Server now correctly offers completions for ``autoxxx`` directive options diff --git a/lib/esbonio/esbonio/lsp/directives.py b/lib/esbonio/esbonio/lsp/directives.py index 025dd604..9966321c 100644 --- a/lib/esbonio/esbonio/lsp/directives.py +++ b/lib/esbonio/esbonio/lsp/directives.py @@ -81,7 +81,8 @@ def discover(self): } self.options = { - k: self.options_to_completion_items(v) for k, v in self.directives.items() + k: self.options_to_completion_items(k, v) + for k, v in self.directives.items() } self.logger.info("Discovered %s directives", len(self.directives)) @@ -258,17 +259,39 @@ def directive_to_completion_item( ), ) - def options_to_completion_items(self, directive: Directive) -> List[CompletionItem]: + def options_to_completion_items( + self, name: str, directive: Directive + ) -> List[CompletionItem]: """Convert a directive's options to a list of completion items. + Unfortunately, the ``autoxxx`` family of directives are a little different. + Each ``autoxxxx`` directive name resolves to the same ``AutodocDirective`` class. + That paricular directive does not have any options, instead the options are + held on the particular Documenter that documents that object type. + + This method does the lookup in order to determine what those options are. + Parameters ---------- + name: + The name of the directive as it appears in an rst file. directive: The directive whose options we are creating completions for. """ options = directive.option_spec + # autoxxx directives require special handlng. + if name.startswith("auto") and self.rst.app: + self.logger.debug("Processing options for '%s' directive", name) + name = name.replace("auto", "") + + self.logger.debug("Documenter name is '%s'", name) + documenter = self.rst.app.registry.documenters.get(name, None) + + if documenter is not None: + options = documenter.option_spec + if options is None: return [] diff --git a/lib/esbonio/tests/data/sphinx-extensions/conf.py b/lib/esbonio/tests/data/sphinx-extensions/conf.py index c75d1b61..9d2bf2c3 100644 --- a/lib/esbonio/tests/data/sphinx-extensions/conf.py +++ b/lib/esbonio/tests/data/sphinx-extensions/conf.py @@ -25,7 +25,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.doctest", "sphinx.ext.intersphinx"] +extensions = ["sphinx.ext.autodoc", "sphinx.ext.doctest", "sphinx.ext.intersphinx"] intersphinx_mapping = { "python": ("https://docs.python.org/3", None), diff --git a/lib/esbonio/tests/lsp/test_directives.py b/lib/esbonio/tests/lsp/test_directives.py index 533cb615..d9d5cdde 100644 --- a/lib/esbonio/tests/lsp/test_directives.py +++ b/lib/esbonio/tests/lsp/test_directives.py @@ -29,6 +29,8 @@ } EXTENSIONS_EXPECTED = { + "autoclass", + "automodule", "py:function", "py:module", "option", @@ -40,8 +42,6 @@ } EXTENSIONS_UNEXPECTED = { - "autoclass", - "automodule", "c:macro", "module", "std:program", @@ -131,6 +131,17 @@ def test_directive_completions(sphinx, project, text, expected, unexpected): completion_test(feature, text, expected, unexpected) +AUTOCLASS_OPTS = { + "members", + "undoc-members", + "noindex", + "inherited-members", + "show-inheritance", + "member-order", + "exclude-members", + "private-members", + "special-members", +} IMAGE_OPTS = {"align", "alt", "class", "height", "scale", "target", "width"} PY_FUNC_OPTS = {"annotation", "async", "module", "noindex", "noindexentry"} C_FUNC_OPTS = {"noindexentry"} @@ -141,6 +152,12 @@ def test_directive_completions(sphinx, project, text, expected, unexpected): [ ("sphinx-default", ".. image:: f.png\n\f :", IMAGE_OPTS, {"ref", "func"}), ("sphinx-default", ".. function:: foo\n\f :", PY_FUNC_OPTS, {"ref", "func"}), + ( + "sphinx-default", + ".. autoclass:: x.y.A\n\f :", + set(), + {"ref", "func"} | AUTOCLASS_OPTS, + ), ( "sphinx-default", " .. image:: f.png\n\f :", @@ -153,6 +170,12 @@ def test_directive_completions(sphinx, project, text, expected, unexpected): PY_FUNC_OPTS, {"ref", "func"}, ), + ( + "sphinx-default", + " .. autoclass:: x.y.A\n\f :", + set(), + {"ref", "func"} | AUTOCLASS_OPTS, + ), ("sphinx-extensions", ".. image:: f.png\n\f :", IMAGE_OPTS, {"ref", "func"}), ( "sphinx-extensions", @@ -160,6 +183,12 @@ def test_directive_completions(sphinx, project, text, expected, unexpected): C_FUNC_OPTS, {"ref", "func"}, ), + ( + "sphinx-extensions", + ".. autoclass:: x.y.A\n\f :", + AUTOCLASS_OPTS, + {"ref", "func"}, + ), ( "sphinx-extensions", " .. image:: f.png\n\f :", @@ -172,6 +201,12 @@ def test_directive_completions(sphinx, project, text, expected, unexpected): C_FUNC_OPTS, {"ref", "func"}, ), + ( + "sphinx-extensions", + " .. autoclass:: x.y.A\n\f :", + AUTOCLASS_OPTS, + {"ref", "func"}, + ), ], ) def test_directive_option_completions(sphinx, project, text, expected, unexpected): From 87716697d5a69748644f9eca771c7513bb6435af Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Wed, 24 Feb 2021 17:40:53 +0000 Subject: [PATCH 40/41] Catch consistency errors. (#112) By running the full build, we are now able to catch errors like invalid references and benefit from Sphinx's caching logic. Also by updating the `PROBLEM_PATTERN` regex, we can catch errors that do not have an associated line number. `DiagnosticList` is a way to ensure we don't get repeat reports of the same issue, but might become a performance issue depending on project scale. Closes #94 Closes #57 --- lib/esbonio/changes/57.feature.rst | 1 + lib/esbonio/changes/94.fix.rst | 1 + lib/esbonio/esbonio/lsp/sphinx.py | 93 ++++++++++++++++++++++------ lib/esbonio/tests/lsp/test_sphinx.py | 63 ++++++++++++++++--- 4 files changed, 130 insertions(+), 28 deletions(-) create mode 100644 lib/esbonio/changes/57.feature.rst create mode 100644 lib/esbonio/changes/94.fix.rst diff --git a/lib/esbonio/changes/57.feature.rst b/lib/esbonio/changes/57.feature.rst new file mode 100644 index 00000000..a8afea5b --- /dev/null +++ b/lib/esbonio/changes/57.feature.rst @@ -0,0 +1 @@ +The language server now reports invalid references as diagnostics diff --git a/lib/esbonio/changes/94.fix.rst b/lib/esbonio/changes/94.fix.rst new file mode 100644 index 00000000..a1a04fb2 --- /dev/null +++ b/lib/esbonio/changes/94.fix.rst @@ -0,0 +1 @@ +Consistency errors are now included in reported diagnostics. diff --git a/lib/esbonio/esbonio/lsp/sphinx.py b/lib/esbonio/esbonio/lsp/sphinx.py index a1d3899e..0f73ca9b 100644 --- a/lib/esbonio/esbonio/lsp/sphinx.py +++ b/lib/esbonio/esbonio/lsp/sphinx.py @@ -1,4 +1,5 @@ """Code for managing sphinx applications.""" +import collections import logging import pathlib import re @@ -20,16 +21,26 @@ from sphinx.domains import Domain from sphinx.util import console -from esbonio.lsp import RstLanguageServer +from esbonio.lsp import LanguageFeature, RstLanguageServer PROBLEM_PATTERN = re.compile( r""" - (?P(.*:\\)?[^:]*):(?P\d+):\s(?P[^:]*):(\s+)?(?P.*) + (?P(.*:\\)?[^:]*): # Capture the path to the file containing the problem + ((?P\d+):)? # Some errors may specify a line number. + \s(?P[^:]*): # Capture the type of error + (\s+)?(?P.*) # Capture the error message """, re.VERBOSE, ) -"""Regular Expression used to identify warnings/errors in Sphinx's output.""" +"""Regular Expression used to identify warnings/errors in Sphinx's output. + +For example:: + + /path/to/file.rst: WARNING: document isn't included in any toctree + /path/to/file.rst:4: WARNING: toctree contains reference to nonexisting document 'changelog', + +""" PROBLEM_SEVERITY = { @@ -38,6 +49,13 @@ } +def get_filepath(uri: str) -> pathlib.Path: + """Given a uri, return the filepath component.""" + + uri = urlparse(uri) + return pathlib.Path(unquote(uri.path)) + + def get_domains(app: Sphinx) -> Iterator[Tuple[str, Domain]]: """Get all the domains registered with an applications. @@ -69,8 +87,7 @@ def get_domains(app: Sphinx) -> Iterator[Tuple[str, Domain]]: def find_conf_py(root_uri: str) -> Optional[pathlib.Path]: """Attempt to find Sphinx's configuration file in the given workspace.""" - uri = urlparse(root_uri) - root = pathlib.Path(unquote(uri.path)) + root = get_filepath(root_uri) # Strangely for windows paths, there's an extra leading slash which we have to # remove ourselves. @@ -89,12 +106,40 @@ def find_conf_py(root_uri: str) -> Optional[pathlib.Path]: return candidate -class SphinxManagement: +class DiagnosticList(collections.UserList): + """A list type dedicated to holding diagnostics. + + This is mainly to ensure that only one instance of a diagnostic ever gets + reported. + """ + + def append(self, item: Diagnostic): + + if not isinstance(item, Diagnostic): + raise TypeError("Expected Diagnostic") + + for existing in self.data: + fields = [ + existing.range == item.range, + existing.message == item.message, + existing.severity == item.severity, + existing.code == item.code, + existing.source == item.source, + ] + + if all(fields): + # Item already added, nothing to do. + return + + self.data.append(item) + + +class SphinxManagement(LanguageFeature): """A LSP Server feature that manages the Sphinx application instance for the project.""" - def __init__(self, rst: RstLanguageServer): - self.rst = rst + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.diagnostics = {} """A place to keep track of diagnostics we can publish to the client.""" @@ -106,7 +151,7 @@ def initialize(self): self.create_app() if self.rst.app is not None: - self.rst.app.builder.read() + self.rst.app.build() def initialized(self): self.report_diagnostics() @@ -116,8 +161,10 @@ def save(self, params: DidSaveTextDocumentParams): if self.rst.app is None: return - self.reset_diagnostics() - self.rst.app.builder.read() + filepath = get_filepath(params.textDocument.uri) + + self.reset_diagnostics(str(filepath)) + self.rst.app.build() self.report_diagnostics() def create_app(self): @@ -172,11 +219,17 @@ def report_diagnostics(self): doc = "/" + doc uri = f"file://{doc}" - self.rst.publish_diagnostics(uri, diagnostics) + self.rst.publish_diagnostics(uri, diagnostics.data) + + def reset_diagnostics(self, filepath: str): + """Reset the list of diagnostics for the given file. - def reset_diagnostics(self): - """Reset the list of diagnostics.""" - self.diagnostics = {filepath: [] for filepath in self.diagnostics.keys()} + Parameters + ---------- + filepath: + The filepath that the diagnostics should be reset for. + """ + self.diagnostics[filepath] = DiagnosticList() def write(self, line): """This method lets us catch output from Sphinx.""" @@ -190,12 +243,16 @@ def write(self, line): diagnostics = self.diagnostics.get(filepath, None) if diagnostics is None: - diagnostics = [] + diagnostics = DiagnosticList() try: line_number = int(match.group("line")) - except ValueError as exc: - self.logger.error("Unable to parse line number", exc) + except (TypeError, ValueError) as exc: + self.logger.debug( + "Unable to parse line number: '%s'", match.group("line") + ) + self.logger.debug(exc) + line_number = 1 range_ = Range(Position(line_number - 1, 0), Position(line_number, 0)) diff --git a/lib/esbonio/tests/lsp/test_sphinx.py b/lib/esbonio/tests/lsp/test_sphinx.py index 7e5d1488..1065137c 100644 --- a/lib/esbonio/tests/lsp/test_sphinx.py +++ b/lib/esbonio/tests/lsp/test_sphinx.py @@ -1,10 +1,18 @@ +import logging import unittest.mock as mock import py.test -from pygls.types import Diagnostic, DiagnosticSeverity, Position, Range +from pygls.types import ( + Diagnostic, + DiagnosticSeverity, + DidSaveTextDocumentParams, + Position, + Range, + TextDocumentIdentifier, +) -from esbonio.lsp.sphinx import SphinxManagement, find_conf_py +from esbonio.lsp.sphinx import DiagnosticList, SphinxManagement, find_conf_py def line(linum: int) -> Range: @@ -128,12 +136,42 @@ def test_find_conf_py(root, candidates, expected): ] }, ), + ( + "/path/to/file.rst: WARNING: document isn't included in any toctree", + { + "/path/to/file.rst": [ + Diagnostic( + line(1), + "document isn't included in any toctree", + severity=DiagnosticSeverity.Warning, + source="sphinx", + ) + ] + }, + ), + ( + "c:\\path\\to\\file.rst: WARNING: document isn't included in any toctree", + { + "c:\\path\\to\\file.rst": [ + Diagnostic( + line(1), + "document isn't included in any toctree", + severity=DiagnosticSeverity.Warning, + source="sphinx", + ) + ] + }, + ), ], ) def test_parse_diagnostics(text, expected): """Ensure that the language server can parse errors from sphinx's output.""" - management = SphinxManagement(mock.Mock()) + rst = mock.Mock() + rst.logger = logging.getLogger("rst") + + file = "" + management = SphinxManagement(rst) management.write(text) # Unfortunately, 'Diagnostic' is just a Python class without an __eq__ implementation @@ -153,8 +191,8 @@ def test_parse_diagnostics(text, expected): assert a.source == b.source # Ensure that can correctly reset diagnostics - management.reset_diagnostics() - assert management.diagnostics == {file: [] for file in expected.keys()} + management.reset_diagnostics(file) + assert len(management.diagnostics[file]) == 0 def test_report_diagnostics(): @@ -167,15 +205,20 @@ def test_report_diagnostics(): manager = SphinxManagement(rst) manager.diagnostics = { - "c:\\Users\\username\\Project\\file.rst": (1, 2, 3), - "/home/username/Project/file.rst": (4, 5, 6), + "c:\\Users\\username\\Project\\file.rst": DiagnosticList([1, 2, 3]), + "/home/username/Project/file.rst": DiagnosticList([4, 5, 6]), } + doc = TextDocumentIdentifier(uri="/some/file.rst") + params = DidSaveTextDocumentParams(text_document=doc, text="") + manager.reset_diagnostics = mock.Mock() - manager.save(None) + manager.save(params) expected = [ - mock.call("file:///c:\\Users\\username\\Project\\file.rst", (1, 2, 3)), - mock.call("file:///home/username/Project/file.rst", (4, 5, 6)), + mock.call( + "file:///c:\\Users\\username\\Project\\file.rst", DiagnosticList([1, 2, 3]) + ), + mock.call("file:///home/username/Project/file.rst", DiagnosticList([4, 5, 6])), ] assert publish_diagnostics.call_args_list == expected From 72fbbf383f410294e5b513e5e08b8594756da6ce Mon Sep 17 00:00:00 2001 From: Alex Carney Date: Thu, 25 Feb 2021 10:42:39 +0000 Subject: [PATCH 41/41] Expose a number of cli arguments (#116) - `--cache-dir`, optionally allows for Language Clients to specify the directory where cached data should be stored. If not given the Language Server will choose a directory based on a hash of the `conf.py`'s filepath. Closes #115 - `--hide-sphinx-output`, allows for Language Clients to prevent Sphinx's build log from being shown. - `--log-filter`, allows for Language Clients to restrict log output to specific loggers. Can be given multiple times. Closes #113 - `--log-level`, allows for Language Clients to specify the verbosity of the log they wish to receive. Closes #87 - `-p`, `--port`, allows for Language Clients to interact with the Language Server over TCP, while setting the port number they wish to use. Closes #114 --- lib/esbonio/changes/113.feature.rst | 4 ++ lib/esbonio/changes/114.feature.rst | 2 + lib/esbonio/changes/115.feature.rst | 2 + lib/esbonio/changes/87.feature.rst | 2 + lib/esbonio/esbonio/__main__.py | 96 +++++++++++++++++++++-------- lib/esbonio/esbonio/lsp/__init__.py | 13 +++- lib/esbonio/esbonio/lsp/sphinx.py | 11 +++- 7 files changed, 101 insertions(+), 29 deletions(-) create mode 100644 lib/esbonio/changes/113.feature.rst create mode 100644 lib/esbonio/changes/114.feature.rst create mode 100644 lib/esbonio/changes/115.feature.rst create mode 100644 lib/esbonio/changes/87.feature.rst diff --git a/lib/esbonio/changes/113.feature.rst b/lib/esbonio/changes/113.feature.rst new file mode 100644 index 00000000..5f3f31fe --- /dev/null +++ b/lib/esbonio/changes/113.feature.rst @@ -0,0 +1,4 @@ +Add ``log-filter`` cli argument that allows Language Clients to choose +which loggers they want to recieve messages from. Also add +``--hide-sphinx-output`` cli argument that can suppress Sphinx's build +log as it it handled separately. diff --git a/lib/esbonio/changes/114.feature.rst b/lib/esbonio/changes/114.feature.rst new file mode 100644 index 00000000..8141b38d --- /dev/null +++ b/lib/esbonio/changes/114.feature.rst @@ -0,0 +1,2 @@ +Add ``-p``, ``--port`` cli arguments that start the Language Server in +TCP mode while specifying the port number to listen on. diff --git a/lib/esbonio/changes/115.feature.rst b/lib/esbonio/changes/115.feature.rst new file mode 100644 index 00000000..a26d5a1c --- /dev/null +++ b/lib/esbonio/changes/115.feature.rst @@ -0,0 +1,2 @@ +Add ``--cache-dir`` cli argument that allows Language Clients to +specify where cached data should be stored e.g. Sphinx's build output. diff --git a/lib/esbonio/changes/87.feature.rst b/lib/esbonio/changes/87.feature.rst new file mode 100644 index 00000000..d6a9c229 --- /dev/null +++ b/lib/esbonio/changes/87.feature.rst @@ -0,0 +1,2 @@ +Add ``--log-level`` cli argument that allows Language Clients to +control the verbosity of the Language Server's log output. diff --git a/lib/esbonio/esbonio/__main__.py b/lib/esbonio/esbonio/__main__.py index 5bf238f5..4fd427da 100644 --- a/lib/esbonio/esbonio/__main__.py +++ b/lib/esbonio/esbonio/__main__.py @@ -5,17 +5,29 @@ import esbonio.lsp as lsp from esbonio import __version__ +from esbonio.lsp import RstLanguageServer from esbonio.lsp.logger import LspHandler -LOG_LEVELS = [logging.ERROR, logging.INFO, logging.DEBUG] +LOG_LEVELS = { + "debug": logging.DEBUG, + "error": logging.ERROR, + "info": logging.INFO, +} -def configure_logging(verbose, server): +class LogFilter: + """A log filter that accepts message from any of the listed logger names.""" - try: - level = LOG_LEVELS[-1] - except IndexError: - level = LOG_LEVELS[-1] + def __init__(self, names): + self.names = names + + def filter(self, record): + return any(record.name == name for name in self.names) + + +def configure_logging(args, server: RstLanguageServer): + + level = LOG_LEVELS[args.log_level] lsp_logger = logging.getLogger("esbonio.lsp") lsp_logger.setLevel(level) @@ -23,41 +35,77 @@ def configure_logging(verbose, server): lsp_handler = LspHandler(server) lsp_handler.setLevel(level) + if args.log_filter is not None: + lsp_handler.addFilter(LogFilter(args.log_filter)) + formatter = logging.Formatter("[%(name)s] %(message)s") lsp_handler.setFormatter(formatter) lsp_logger.addHandler(lsp_handler) - sphinx_logger = logging.getLogger("esbonio.sphinx") - sphinx_logger.setLevel(level) + if not args.hide_sphinx_output: + sphinx_logger = logging.getLogger("esbonio.sphinx") + sphinx_logger.setLevel(logging.INFO) - sphinx_handler = LspHandler(server) - sphinx_handler.setLevel(level) + sphinx_handler = LspHandler(server) + sphinx_handler.setLevel(logging.INFO) - formatter = logging.Formatter("%(message)s") - sphinx_handler.setFormatter(formatter) - sphinx_logger.addHandler(sphinx_handler) + formatter = logging.Formatter("%(message)s") + sphinx_handler.setFormatter(formatter) + sphinx_logger.addHandler(sphinx_handler) -def start_server(verbose): +def start_server(args): """Start the language server.""" - server = lsp.create_language_server(lsp.BUILTIN_MODULES) - configure_logging(verbose, server) - server.start_io() + server = lsp.create_language_server(lsp.BUILTIN_MODULES, cache_dir=args.cache_dir) + configure_logging(args, server) + + if args.port: + server.start_tcp("localhost", args.port) + else: + server.start_io() cli = argparse.ArgumentParser(prog="esbonio", description="The Esbonio language server") + cli.add_argument( - "--version", action="store_true", help="print the current version and exit" + "--cache-dir", + default=None, + type=str, + help="the directory where cached data should be stored, e.g. Sphinx build output ", ) + cli.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="increase output verbosity, repeatable e.g. -v, -vv, -vvv, ...", + "--hide-sphinx-output", + action="store_true", + help="hide sphinx build output from the log", ) +cli.add_argument( + "--log-filter", + action="append", + help="only include log messages from loggers with the given name," + + "can be set multiple times.", +) + +cli.add_argument( + "--log-level", + choices=["error", "info", "debug"], + default="error", + help="set the level of log message to show from the language server", +) + +cli.add_argument( + "-p", + "--port", + type=int, + default=None, + help="start a TCP instance of the language server listening on the given port ", +) + +cli.add_argument( + "--version", action="store_true", help="print the current version and exit" +) args = cli.parse_args() @@ -65,4 +113,4 @@ def start_server(verbose): print("v{}".format(__version__)) sys.exit(0) -start_server(args.verbose) +start_server(args) diff --git a/lib/esbonio/esbonio/lsp/__init__.py b/lib/esbonio/esbonio/lsp/__init__.py index d24617d1..f0c4dfa2 100644 --- a/lib/esbonio/esbonio/lsp/__init__.py +++ b/lib/esbonio/esbonio/lsp/__init__.py @@ -37,9 +37,12 @@ def __init__(self, rst: "RstLanguageServer"): class RstLanguageServer(LanguageServer): - def __init__(self, *args, **kwargs): + def __init__(self, cache_dir=None, *args, **kwargs): super().__init__(*args, **kwargs) + self.cache_dir = cache_dir + """The folder to store cached data in.""" + self.logger = logging.getLogger(__name__) """The logger that should be used for all Language Server log entries""" @@ -121,15 +124,19 @@ def default(obj): return json.dumps(obj, default=default) -def create_language_server(modules: List[str]) -> RstLanguageServer: +def create_language_server( + modules: List[str], cache_dir: Optional[str] = None +) -> RstLanguageServer: """Create a new language server instance. Parameters ---------- modules: The list of modules that should be loaded. + cache_dir: + The folder to use for cached data. """ - server = RstLanguageServer() + server = RstLanguageServer(cache_dir) for mod in modules: server.load_module(mod) diff --git a/lib/esbonio/esbonio/lsp/sphinx.py b/lib/esbonio/esbonio/lsp/sphinx.py index 0f73ca9b..4ef2d554 100644 --- a/lib/esbonio/esbonio/lsp/sphinx.py +++ b/lib/esbonio/esbonio/lsp/sphinx.py @@ -1,5 +1,6 @@ """Code for managing sphinx applications.""" import collections +import hashlib import logging import pathlib import re @@ -182,8 +183,14 @@ def create_app(self): src = conf_py.parent - # TODO: Create a unique scratch space based on the project. - build = appdirs.user_cache_dir("esbonio", "swyddfa") + if self.rst.cache_dir is not None: + build = self.rst.cache_dir + else: + # Try to pick a sensible dir based on the project's location + cache = appdirs.user_cache_dir("esbonio", "swyddfa") + project = hashlib.md5(str(src).encode()).hexdigest() + build = pathlib.Path(cache) / project + doctrees = pathlib.Path(build) / "doctrees" self.rst.logger.debug("Config dir %s", src)