diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1008006..af6d16e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,17 +1,4 @@ -.. _changelog: - ---------- -Changelog ---------- - -.. note:: - - Until the Python API has stabilised, ``venvstacks`` is using - `ZeroVer `__ (starting from 0.1.0). - - Refer to :ref:`version-numbering` for additional details - on the way releases are versioned. - +.. Included in published docs via docs/changelog.rst Unreleased ========== @@ -20,7 +7,6 @@ See the fragment files in the `changelog.d directory`_. .. _changelog.d directory: https://github.com/lmstudio-ai/venvstacks/tree/main/changelog.d - .. scriv-insert-here .. _changelog-0.1.1: @@ -31,7 +17,8 @@ See the fragment files in the `changelog.d directory`_. Changed ------- -- Update docs URL to `https://venvstacks.lmstudio.ai`__ +- Update docs URL to + `https://venvstacks.lmstudio.ai `__ - Add OpenGraph metadata to docs landing page @@ -75,6 +62,6 @@ Changed Added ----- -- Initial export of `venvstacks` from Project Amphibian. +- Initial export of ``venvstacks`` from Project Amphibian. - Adopted ``scriv`` for ``CHANGELOG`` management. diff --git a/docs/api/cli/index.rst b/docs/api/cli/index.rst index 14528fd..0065293 100644 --- a/docs/api/cli/index.rst +++ b/docs/api/cli/index.rst @@ -1,6 +1,12 @@ venvstacks.cli ============== +.. meta:: + :og:title: venvstacks.cli API - venvstacks Documentation + :og:type: website + :og:url: https://venvstacks.lmstudio.ai/api/cli/ + :og:description: venvstacks.cli Python API - venvstacks Documentation + .. warning:: The Python API is *NOT YET STABLE*. diff --git a/docs/api/index.rst b/docs/api/index.rst index 4858fdc..68baf26 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -2,6 +2,12 @@ Python API ---------- +.. meta:: + :og:title: venvstacks API - venvstacks Documentation + :og:type: website + :og:url: https://venvstacks.lmstudio.ai/api/ + :og:description: venvstacks Python API - venvstacks Documentation + .. warning:: The Python API is *NOT YET STABLE*. diff --git a/docs/api/pack_venv/index.rst b/docs/api/pack_venv/index.rst index f9f216c..beae01f 100644 --- a/docs/api/pack_venv/index.rst +++ b/docs/api/pack_venv/index.rst @@ -1,6 +1,12 @@ venvstacks.pack\_venv ===================== +.. meta:: + :og:title: venvstacks.pack_venv API - venvstacks Documentation + :og:type: website + :og:url: https://venvstacks.lmstudio.ai/api/pack_venv/ + :og:description: venvstacks.pack_venv Python API - venvstacks Documentation + .. warning:: The Python API is *NOT YET STABLE*. diff --git a/docs/api/stacks/index.rst b/docs/api/stacks/index.rst index 9cc8cef..1743d17 100644 --- a/docs/api/stacks/index.rst +++ b/docs/api/stacks/index.rst @@ -1,6 +1,24 @@ venvstacks.stacks ================= +.. meta:: + :og:title: venvstacks.stacks API - venvstacks Documentation + :og:type: website + :og:url: https://venvstacks.lmstudio.ai/api/stacks/ + :og:description: venvstacks.stacks Python API - venvstacks Documentation + +.. TODO: Replace the autosummary tables with: + * inline docs for the high level interface and the exceptions + * a stacks/metadata-details/ page + * a stacks/archive-publication/ page + * a stacks/local-exports/ page + * a stacks/layer-specifications/ page + * a stacks/layer-build-environments/ page + * a stacks/build-configuration/ page + + Dedicated pages correspond to the sections below (except as noted). + Page names contain hyphens to ensure they're not valid submodule names. + .. warning:: The Python API is *NOT YET STABLE*. diff --git a/changelog.d/20241105_141935_ncoghlan_more_resilient_postinstall.rst b/docs/changelog.d/20241105_141935_ncoghlan_more_resilient_postinstall.rst similarity index 100% rename from changelog.d/20241105_141935_ncoghlan_more_resilient_postinstall.rst rename to docs/changelog.d/20241105_141935_ncoghlan_more_resilient_postinstall.rst diff --git a/changelog.d/20241108_162952_ncoghlan_fix_automatic_layer_versioning.rst b/docs/changelog.d/20241108_162952_ncoghlan_fix_automatic_layer_versioning.rst similarity index 100% rename from changelog.d/20241108_162952_ncoghlan_fix_automatic_layer_versioning.rst rename to docs/changelog.d/20241108_162952_ncoghlan_fix_automatic_layer_versioning.rst diff --git a/changelog.d/20241108_215625_ncoghlan_enable_docstring_style_checks.rst b/docs/changelog.d/20241108_215625_ncoghlan_enable_docstring_style_checks.rst similarity index 100% rename from changelog.d/20241108_215625_ncoghlan_enable_docstring_style_checks.rst rename to docs/changelog.d/20241108_215625_ncoghlan_enable_docstring_style_checks.rst diff --git a/docs/changelog.d/20241111_104113_ncoghlan_add_stack_format_docs.rst b/docs/changelog.d/20241111_104113_ncoghlan_add_stack_format_docs.rst new file mode 100644 index 0000000..1727084 --- /dev/null +++ b/docs/changelog.d/20241111_104113_ncoghlan_add_stack_format_docs.rst @@ -0,0 +1,5 @@ +Added +----- + +- Added documentation for the :ref:`stack-specification-format`. + diff --git a/changelog.d/README.txt b/docs/changelog.d/README.txt similarity index 100% rename from changelog.d/README.txt rename to docs/changelog.d/README.txt diff --git a/changelog.d/ghrel_template.md.j2 b/docs/changelog.d/ghrel_template.md.j2 similarity index 100% rename from changelog.d/ghrel_template.md.j2 rename to docs/changelog.d/ghrel_template.md.j2 diff --git a/changelog.d/new_fragment.rst.j2 b/docs/changelog.d/new_fragment.rst.j2 similarity index 88% rename from changelog.d/new_fragment.rst.j2 rename to docs/changelog.d/new_fragment.rst.j2 index 7bf9b20..1523544 100644 --- a/changelog.d/new_fragment.rst.j2 +++ b/docs/changelog.d/new_fragment.rst.j2 @@ -10,7 +10,7 @@ .. "resolved" works for most categories, but replace the verb as necessary. .. Use `:pr:`NN` to refer to pull requests. Other Sphinx roles also work here. .. -.. Deleting this header after editing is recommended. +.. Deleting this header after editing is recommended (it contains 16 lines). Category -------- diff --git a/docs/changelog.rst b/docs/changelog.rst index 565b052..d38c6b0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1 +1,31 @@ +.. _changelog: + +--------- +Changelog +--------- + +.. meta:: + :og:title: venvstacks Changelog - venvstacks Documentation + :og:type: website + :og:url: https://venvstacks.lmstudio.ai/changelog/ + :og:description: venvstacks Changelog - venvstacks Documentation + +.. note:: + + Until the Python API has stabilised, ``venvstacks`` is using + `ZeroVer `__ (starting from 0.1.0). + + Refer to :ref:`version-numbering` for additional details + on the way releases are versioned. + +.. Syntax check the changelog fragments + +.. toctree:: + :hidden: + :glob: + + changelog.d/* + +.. Include the scriv-generated changelog details + .. include:: ../CHANGELOG.rst diff --git a/docs/conf.py b/docs/conf.py index 603e4c2..793a34f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,3 +1,5 @@ +"""Sphinx configuration for venvstacks documentation.""" + # Configuration file for the Sphinx documentation builder. # # For the full list of built-in configuration values, see the documentation: @@ -28,12 +30,16 @@ templates_path = ["_templates"] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +# Allow Markdown-style single backtick syntax for inline literals +default_role = "literal" # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = "furo" html_static_path = ["_static"] +pygments_style = "sphinx" +pygments_dark_style = "monokai" # Docs are published directly to GitHub pages, consider them to be unversioned html_title = "venvstacks documentation" @@ -57,12 +63,14 @@ "issue": ("https://github.com/lmstudio-ai/venvstacks/issues/%s", "#%s"), "pr": ("https://github.com/lmstudio-ai/venvstacks/pull/%s", "PR #%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), + "toml": ("https://toml.io/en/v1.0.0#%s", "%s"), } +extlinks_detect_hardcoded_links = True # -- Options for intersphinx ---------------------------------------------------------- intersphinx_mapping = { - "py": ("https://docs.python.org/3", None), + "pystd": ("https://docs.python.org/3", None), "packaging": ("https://packaging.python.org/en/latest/", None), } diff --git a/docs/design.rst b/docs/design.rst index 02666c8..9ec5334 100644 --- a/docs/design.rst +++ b/docs/design.rst @@ -2,6 +2,12 @@ Design Discussion ----------------- +.. meta:: + :og:title: venvstacks Design - venvstacks Documentation + :og:type: website + :og:url: https://venvstacks.lmstudio.ai/design/ + :og:description: venvstacks Design Discussion - venvstacks Documentation + Project ======= diff --git a/docs/development/index.rst b/docs/development/index.rst index acb9a27..c5aebaa 100644 --- a/docs/development/index.rst +++ b/docs/development/index.rst @@ -4,6 +4,12 @@ Development ----------- +.. meta:: + :og:title: venvstacks Development - venvstacks Documentation + :og:type: website + :og:url: https://venvstacks.lmstudio.ai/development/ + :og:description: venvstacks Developer Guide - venvstacks Documentation + Getting Started =============== @@ -12,7 +18,8 @@ Getting Started This document aims to get you setup to work on venvstacks and to act as a guide and reference to the development setup. If you face any issues during this -process, please `open an issue`_ about it on the issue tracker. +process, please :issue:`open an issue ` +about it on the issue tracker. Get the source code @@ -279,7 +286,6 @@ After release: * Bump the version in ``pyproject.toml`` and add a ``.dev0`` suffix .. _`Getting Started`: https://pip.pypa.io/en/stable/development/getting-started/ -.. _`open an issue`: https://github.com/lmstudio-ai/venvstacks/issues/new?title=Trouble+with+development+environment .. _`rich CLI`: https://docs.pytest.org/en/stable/how-to/usage.html#specifying-which-tests-to-run .. _`GitHub`: https://github.com/lmstudio-ai/venvstacks .. _`testing README file`: https://github.com/lmstudio-ai/venvstacks/blob/main/tests/README.md diff --git a/docs/file-formats.rst b/docs/file-formats.rst new file mode 100644 index 0000000..bbce47e --- /dev/null +++ b/docs/file-formats.rst @@ -0,0 +1,242 @@ +------------------------------ +Environment Stack File Formats +------------------------------ + +.. meta:: + :og:title: venvstacks File Formats - venvstacks Documentation + :og:type: website + :og:url: https://venvstacks.lmstudio.ai/stack-format/ + :og:description: venvstacks Specification and Metadata File Formats - venvstacks Documentation + + +.. _stack-specification-format: + +File naming and formats +======================= + +By convention, virtual environment stacks are specified in a file named ``venvstacks.toml``. + +The default output folder for layer metadata when publishing artifacts and locally exporting +environments is called ``__venvstacks__``. The platform-specific layer summary metadata +files are called ``venvstacks.json`` and each is written to a folder named after the target +platform in the parent metadata folder. The per-layer metadata files are written to an +``env_metadata`` folder within the platform folders. +Refer to :ref:`layer-metadata` for additional details. + +The layer configuration metadata within deployed environments is written to +``share/venv/metadata/venvstacks_layer.json``. +Refer to :ref:`deployed-layer-config` for additional details. + +All human-edited input files are written using `TOML `__, as this is a file +format that combines the runtime simplicity and Unicode text compatibility of JSON with the +line-oriented human friendliness of the classic ``.ini`` format. It is the same config file +syntax used to define ``pyproject.toml`` when +:external+packaging:ref:`publishing Python packages `. + +All output metadata files generated by the build process are emitted as `JSON `__. + +Defining virtual environment stacks +=================================== + +Virtual environment stacks are defined using the following top-level fields, which are all TOML +:toml:`arrays of tables `: + +* ``[[runtimes]]`` +* ``[[frameworks]]`` +* ``[[applications]]`` + +Common layer specification fields +--------------------------------- + +All layer specifications must contain the following two fields: + +* ``name`` (:toml:`string`): the name of the layer being specified +* ``requirements`` (:toml:`array` of :toml:`strings `): + the top-level Python distribution packages to be installed as part of this layer. + Dependencies are declared using the standard Python + :external+packaging:ref:`dependency specifier ` format. + These declared dependencies will be transitively locked when locking the layer. + The list of requirements must be present, but is permitted to be empty. + +All layer specifications may also contain the following optional fields: + +* ``platforms`` (:toml:`array` of :toml:`strings `): + by default, all layers are built for all target platforms. Setting this field + allows the layer build to be narrowed to a subset of the supported targets. + Setting this field to an empty list also allows a layer build to be disabled + without having to delete it entirely. + Permitted entries in the ``platforms`` list are: + + * ``"win_amd64"``: Windows on x86-64 + * ``"linux_x86_64"``: Linux on x86_64 + * ``"macosx_arm64"``: macOS on Apple silicon + * ``"macosx_x86_64"``: macOS on Intel silicon (not currently tested in CI) + +* ``versioned`` (:toml:`boolean`): by default, and when this setting is ``False``, + the layer is considered either unversioned or explicitly versioned + (depending on whether or not a version number is included in the layer name). + The layer metadata will always report the lock version for these layers as + ``1`` and this value is never implicitly included when deriving other names + from the layer name. + When this setting is ``True``, the layer is considered implicitly versioned. + For implicitly versioned layers, a lock version number is stored as part of + the environment lock metadata, and automatically incremented when the + environment lock file changes as the result of a layer locking request. + The layer metadata will report the saved lock version for implicitly versioned + layers and this value is automatically included when deriving some other names + from the layer name. + +The following layer versioning styles are supported: + +* *explicitly versioned*: layer name uses a format like ``cpython@3.12``, where + the layer "version" is considered part of the layer name. Dependencies from + other layers must refer to the specific version. The ``versioned`` field should be + omitted or explicitly set to ``False``. Explicit versioning allows upper layers + to depend on different versions of the "same" lower layer, but also requires + those layers to be explicitly migrated to new versions of the lower layer. + Explicit versioning also allows multiple versions of the "same" layer to be + built and published in parallel. + +* *implicitly versioned*: layer name uses a format like ``scipy`` with ``versioned`` + set to ``True``. Dependencies from other layers refer to the unversioned layer name, + and are automatically updated to depend on the new version of the lower layer when + the locked requirements change. Some component names derived from the layer name + will be implicitly rewritten to use ``"{layer_name}@{lock_version}"`` rather than + using the layer name on its own. Only the latest version of an implicitly versioned + layer can be built and published, but different versions can still be installed + in parallel on target systems. + +* *unversioned*: layer name uses a format like ``my-app`` with ``versioned`` + omitted or set to ``False``. Dependencies from other layers refer to the + unversioned layer name. Only the latest version of an implicitly versioned + layer can be built and published, and only one versioned can be installed + on any given target system. :ref:`Artifact tagging ` allows multiple versions + of unversioned layers to still be distributed in parallel. + +Refer to :ref:`layer-names` for additional details on how layer names are used +when building virtual environment stacks, as well as aspects to consider when +choosing between unversioned, explicitly versioned, and implicitly versioned layers. + + +Runtime layer specification fields +---------------------------------- + +Runtime layer specifications must contain the following additional field: + +* ``fully_versioned_name`` (:toml:`string`): the :pypi:`pbs-installer` name + of the Python runtime to be installed as the base runtime for this layer + (and any upper layers that depend on this layer). + + +Framework layer specification fields +------------------------------------ + +Framework layer specifications must contain the following additional field: + +* ``runtime`` (:toml:`string`): the name of the runtime layer that this framework layer uses. + +On platforms which use symlinks between layered environments and their base +environments (any platform other than Windows), the runtime layer name will +be recorded in the ``runtime_name`` field of the framework layer metadata. +This allows for transparent security updates of the base runtime layer (for +example, to update to new OpenSSL versions or CPython maintenance releases), +without needing to republish the upper layers that use that base runtime. + +On Windows, where some elements of the base runtime are copied into each +layered environment that depends on it, the runtime ``fully_versioned_name`` +field will be recorded in the ``runtime_name`` field of the framework layer +metadata. This still allows for transparent security updates of the base +runtime layer in (for +example, to update to new OpenSSL versions or CPython maintenance releases), +without needing to republish the upper layers that use that base runtime. + +.. warning:: The current handling of the ``runtime_name`` field in the layer + metadata is highly questionable, and hence subject to change in + future releases without a deprecation period. + + +Application layer specification fields +-------------------------------------- + +Application layer specifications must contain the following additional field: + +* ``frameworks`` (:toml:`array` of :toml:`strings `): + the names of the framework layers that this application layer uses. + +Application layer specifications may also contain the following additional field: + +* ``launch_module`` (:toml:`string`): a relative path (starting from the folder containing + the stack specification file) that specifies a Python module or import package that should + be included in the built environment for execution with the :option:`-m` switch. + +The ``runtime`` dependency for application layers is not specified directly. Instead, all +of the declared framework dependencies *must* depend on the same runtime layer, and that +base runtime also becomes the base runtime for the application layer using those frameworks. + + +.. note:: updating the launch module contents does *not* implicitly update the lock version + for implicitly versioned environments (but it does update the ``archive_build`` + field for published artifacts). + + +.. _layer-names: + +Layer names and versioning +-------------------------- + +Regardless of how a layer is versioned, the layer name is used directly +(with no additional prefix or suffix) when referring to the layer as a +dependency in another layer specification. + +The layer name is also used directly (in combination with the :term:`layer type` +prefix) for the following purposes: + +* the name of the layer build environment +* the name of the layer requirements file folder +* as part of the name of the transitively locked layer requirements files +* as the base name for the layer environment metadata file emitted when + publishing or exporting the environment +* as the ``layer_name`` field in the generated layer metadata + +Runtime layers do not have a layer type prefix, while framework and application +layers use ``app-*`` and ``framework-*`` respectively. + +Explicitly versioned and unversioned layers use their layer name directly + (in combination with their :term:`layer type` prefix) for the following purposes: + +* the name of the deployed layer environment when publishing artifacts or + locally exporting environments +* as the ``install_target`` field in the generated layer metadata +* when referring to the layer as a dependency in another layer's deployment + configuration and output metadata + +Implicitly versioned layers will instead use ``"{layer_name}@{lock_version}"`` +for these deployment related purposes. + + +.. note:: A future documentation update will provide additional guidance on the trade-offs + between explicit versioning, implicit versioning, and leaving layers unversioned. + + +.. _layer-requirements: + +Locked layer requirements +========================= + +.. note:: A future documentation update will cover the ``venvstacks lock`` output files here. + +.. _deployed-layer-config: + +Deployed layer configuration +============================ + +.. note:: A future documentation update will cover the ``share/venv/metadata/venvstacks_layer.json`` files here. + +.. _layer-metadata: + +Published layer metadata +======================== + +.. note:: A future documentation update will cover the ``venvstacks publish`` + and ``venvstacks local-export`` output metadata files here, including + the effects of the ``--tag-outputs`` command line option when publishing. diff --git a/docs/glossary.rst b/docs/glossary.rst index 0d9275a..7e54ba9 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -1,7 +1,15 @@ +.. _glossary: + ---------------------------- Essential Terms and Concepts ---------------------------- +.. meta:: + :og:title: venvstacks Glossary - venvstacks Documentation + :og:type: website + :og:url: https://venvstacks.lmstudio.ai/glossary/ + :og:description: venvstacks Glossary - venvstacks Documentation + .. glossary:: archive @@ -60,6 +68,17 @@ Essential Terms and Concepts applications which embedding applications will invoke. Applications depend on one or more framework layers + layer category + layer kind + layer type + These terms all refer to the different categories of :term:`layer` that + ``venvstacks`` defines. In the code, ``kind`` refers to the singular + forms (``runtime``, ``framework``, ``application``), while ``category`` + refers to the plural forms (``runtimes``, ``frameworks``, ``applications``). + The documentation will generally refer to layer types rather than layer + kinds (the code avoids this usage due to the potential confusion with + Python object types). + stack environment stack An application layer with its supporting framework and base runtime layers. diff --git a/docs/index.rst b/docs/index.rst index 90d6a55..ee5a59e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ to chain together three layers of Python virtual environments: * "Application" layers: environments containing components to be launched directly Application layer environments may include additional unpackaged Python launch modules or -packages for invocation with ``python``'s :py:option:`-m` switch. +packages for invocation with ``python``'s :option:`-m` switch. While the layers are archived and published separately, their dependency locking is integrated, allowing the application layers to share dependencies installed in the framework layers, @@ -54,6 +54,7 @@ make future Python version upgrades more irritating): overview glossary + file-formats design api/index development/index diff --git a/docs/overview.rst b/docs/overview.rst index 8d9c79f..a5fe839 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -4,6 +4,12 @@ Project Overview ---------------- +.. meta:: + :og:title: venvstacks Overview - venvstacks Documentation + :og:type: website + :og:url: https://venvstacks.lmstudio.ai/overview/ + :og:description: venvstacks Project Overview - venvstacks Documentation + Command line interface ====================== diff --git a/pyproject.toml b/pyproject.toml index c1fd7e7..fd6cb1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,4 +142,5 @@ convention = "google" [tool.scriv] version = "literal: pyproject.toml: project.version" +fragment_directory = "docs/changelog.d" ghrel_template = "file: ghrel_template.md.j2" diff --git a/src/venvstacks/stacks.py b/src/venvstacks/stacks.py index 62af817..ca96642 100755 --- a/src/venvstacks/stacks.py +++ b/src/venvstacks/stacks.py @@ -1,7 +1,7 @@ #!/bin/python3 """Portable Python environment stacks. -Creates Python runtime, framework, and app environments based on venvstacks.toml +Creates Python runtime, framework, and app environments based on ``venvstacks.toml`` """ import hashlib diff --git a/tox.ini b/tox.ini index 4fdbf76..e1a447d 100644 --- a/tox.ini +++ b/tox.ini @@ -45,18 +45,18 @@ commands = mypy --strict --exclude 'tests/sample_project' {posargs} src/ tests/ misc/ [testenv:docs] -groups = [] +groups = deps = -r docs/requirements.txt allowlist_externals = sphinx-build commands = - sphinx-build -b dirhtml {posargs} docs/ docs/_build + sphinx-build -W -b dirhtml {posargs} docs/ docs/_build [testenv:linkcheck] -groups = [] +groups = deps = -r docs/requirements.txt allowlist_externals = sphinx-build commands = - sphinx-build -b linkcheck {posargs} docs/ docs/_build + sphinx-build -W -b linkcheck {posargs} docs/ docs/_build [gh] python =