Merge pull request #28 from watermarkhu/development

Better find, caching, documentation

.github/workflows/docs.yml

@@ -0,0 +1,42 @@
+name: documentation
+
+on: 
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths: 
+      - 'docs/**'
+
+permissions:
+  contents: write
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v3
+
+      - name: Install dependencies
+        run: |
+          poetry install --without test,dev
+
+      - name: Sphinx build
+        run: |
+          cd docs && make html
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+        with:
+          publish_branch: gh-pages
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: docs/_build/
+          force_orphan: true

.gitignore

@@ -160,4 +160,7 @@ cython_debug/
 #.idea/
 
 # Ruff cache
-.ruff_cache/
+.ruff_cache/
+
+# Textemate cache
+.textmate_cache/

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.2.2
+    rev: v0.3.0
     hooks:
       # Run the linter.
       - id: ruff
@@ -19,5 +19,6 @@ repos:
       exclude: |
         (?x)^(
             example.py|
-            test\/.*.py
+            test\/.*.py|
+            docs\/.*.py
         )$

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Current File",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": true
+        }
+    ]
+}

README.md

@@ -9,7 +9,7 @@
 
 # textmate-grammar-python
 
-An interpreter for grammar files as defined by TextMate and used in VSCode, implemented in Python. TextMate grammars use the oniguruma dialect (https://github.com/kkos/oniguruma). Supports loading grammar files from JSON, PLIST, or YAML format. 
+A lexer and tokenizer for grammar files as defined by TextMate and used in VSCode, implemented in Python. TextMate grammars use the oniguruma dialect (https://github.com/kkos/oniguruma). Supports loading grammar files from JSON, PLIST, or YAML format. 
 
 
 ## Usage
@@ -65,8 +65,3 @@ Alternatively, with the keyword argument `flatten` the element is displayed as a
 ## TODO
 - Implement Begin/While pattern, required for other grammars.
 
-## Sources
-- [Textmate guide](https://www.apeth.com/nonblog/stories/textmatebundle.html)
-- [VSCode Syntax Highlighting guide](https://code.visualstudio.com/api/language-extensions/syntax-highlight-guide)
-- [vscode-textmate](https://github.com/microsoft/vscode-textmate)
-- [Macromates texmate](https://macromates.com/textmate/manual/)

docs/.gitignore

@@ -0,0 +1 @@
+./_build

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/apidocs/index.rst

@@ -0,0 +1,11 @@
+API Reference
+=============
+
+This page contains auto-generated API reference documentation [#f1]_.
+
+.. toctree::
+   :titlesonly:
+
+   textmate_grammar/textmate_grammar
+
+.. [#f1] Created with `sphinx-autodoc2 <https://github.com/chrisjsewell/sphinx-autodoc2>`_

docs/apidocs/textmate_grammar/textmate_grammar.cache.rst

@@ -0,0 +1,147 @@
+:py:mod:`textmate_grammar.cache`
+================================
+
+.. py:module:: textmate_grammar.cache
+
+.. autodoc2-docstring:: textmate_grammar.cache
+   :allowtitles:
+
+Module Contents
+---------------
+
+Classes
+~~~~~~~
+
+.. list-table::
+   :class: autosummary longtable
+   :align: left
+
+   * - :py:obj:`TextmateCache <textmate_grammar.cache.TextmateCache>`
+     - .. autodoc2-docstring:: textmate_grammar.cache.TextmateCache
+          :summary:
+   * - :py:obj:`SimpleCache <textmate_grammar.cache.SimpleCache>`
+     - .. autodoc2-docstring:: textmate_grammar.cache.SimpleCache
+          :summary:
+   * - :py:obj:`ShelveCache <textmate_grammar.cache.ShelveCache>`
+     - .. autodoc2-docstring:: textmate_grammar.cache.ShelveCache
+          :summary:
+
+Functions
+~~~~~~~~~
+
+.. list-table::
+   :class: autosummary longtable
+   :align: left
+
+   * - :py:obj:`init_cache <textmate_grammar.cache.init_cache>`
+     - .. autodoc2-docstring:: textmate_grammar.cache.init_cache
+          :summary:
+
+Data
+~~~~
+
+.. list-table::
+   :class: autosummary longtable
+   :align: left
+
+   * - :py:obj:`CACHE_DIR <textmate_grammar.cache.CACHE_DIR>`
+     - .. autodoc2-docstring:: textmate_grammar.cache.CACHE_DIR
+          :summary:
+   * - :py:obj:`CACHE <textmate_grammar.cache.CACHE>`
+     - .. autodoc2-docstring:: textmate_grammar.cache.CACHE
+          :summary:
+
+API
+~~~
+
+.. py:data:: CACHE_DIR
+   :canonical: textmate_grammar.cache.CACHE_DIR
+   :value: 'resolve(...)'
+
+   .. autodoc2-docstring:: textmate_grammar.cache.CACHE_DIR
+
+.. py:class:: TextmateCache
+   :canonical: textmate_grammar.cache.TextmateCache
+
+   Bases: :py:obj:`typing.Protocol`
+
+   .. autodoc2-docstring:: textmate_grammar.cache.TextmateCache
+
+   .. py:method:: cache_valid(filepath: pathlib.Path) -> bool
+      :canonical: textmate_grammar.cache.TextmateCache.cache_valid
+
+      .. autodoc2-docstring:: textmate_grammar.cache.TextmateCache.cache_valid
+
+   .. py:method:: load(filepath: pathlib.Path) -> textmate_grammar.elements.ContentElement
+      :canonical: textmate_grammar.cache.TextmateCache.load
+
+      .. autodoc2-docstring:: textmate_grammar.cache.TextmateCache.load
+
+   .. py:method:: save(filePath: pathlib.Path, element: textmate_grammar.elements.ContentElement) -> None
+      :canonical: textmate_grammar.cache.TextmateCache.save
+
+      .. autodoc2-docstring:: textmate_grammar.cache.TextmateCache.save
+
+.. py:class:: SimpleCache()
+   :canonical: textmate_grammar.cache.SimpleCache
+
+   Bases: :py:obj:`textmate_grammar.cache.TextmateCache`
+
+   .. autodoc2-docstring:: textmate_grammar.cache.SimpleCache
+
+   .. rubric:: Initialization
+
+   .. autodoc2-docstring:: textmate_grammar.cache.SimpleCache.__init__
+
+   .. py:method:: cache_valid(filepath: pathlib.Path) -> bool
+      :canonical: textmate_grammar.cache.SimpleCache.cache_valid
+
+      .. autodoc2-docstring:: textmate_grammar.cache.SimpleCache.cache_valid
+
+   .. py:method:: load(filepath: pathlib.Path) -> textmate_grammar.elements.ContentElement
+      :canonical: textmate_grammar.cache.SimpleCache.load
+
+      .. autodoc2-docstring:: textmate_grammar.cache.SimpleCache.load
+
+   .. py:method:: save(filepath: pathlib.Path, element: textmate_grammar.elements.ContentElement) -> None
+      :canonical: textmate_grammar.cache.SimpleCache.save
+
+      .. autodoc2-docstring:: textmate_grammar.cache.SimpleCache.save
+
+.. py:class:: ShelveCache()
+   :canonical: textmate_grammar.cache.ShelveCache
+
+   Bases: :py:obj:`textmate_grammar.cache.TextmateCache`
+
+   .. autodoc2-docstring:: textmate_grammar.cache.ShelveCache
+
+   .. rubric:: Initialization
+
+   .. autodoc2-docstring:: textmate_grammar.cache.ShelveCache.__init__
+
+   .. py:method:: cache_valid(filepath: pathlib.Path) -> bool
+      :canonical: textmate_grammar.cache.ShelveCache.cache_valid
+
+      .. autodoc2-docstring:: textmate_grammar.cache.ShelveCache.cache_valid
+
+   .. py:method:: load(filepath: pathlib.Path) -> textmate_grammar.elements.ContentElement
+      :canonical: textmate_grammar.cache.ShelveCache.load
+
+      .. autodoc2-docstring:: textmate_grammar.cache.ShelveCache.load
+
+   .. py:method:: save(filepath: pathlib.Path, element: textmate_grammar.elements.ContentElement) -> None
+      :canonical: textmate_grammar.cache.ShelveCache.save
+
+      .. autodoc2-docstring:: textmate_grammar.cache.ShelveCache.save
+
+.. py:data:: CACHE
+   :canonical: textmate_grammar.cache.CACHE
+   :type: textmate_grammar.cache.TextmateCache
+   :value: 'SimpleCache(...)'
+
+   .. autodoc2-docstring:: textmate_grammar.cache.CACHE
+
+.. py:function:: init_cache(type: str = 'simple') -> textmate_grammar.cache.TextmateCache
+   :canonical: textmate_grammar.cache.init_cache
+
+   .. autodoc2-docstring:: textmate_grammar.cache.init_cache