feat: Pydantic model for the serialization schema (#888)

This PR ports the pydantic definition from guppy (including
CQCL/guppylang#172), and updates it to sync up
with the current schema.

Changes to the schema:

- Renamed the `InputExtensions` class to `ExtensionSet`
- Added a named definition for `Const`
- Dropped `DummyOp`. It was referenced in a couple field types, but the
model was never defined.

The main difference with the model in `guppy` is the use of
`RootModel`-based classes instead of variables assigned to a
`TypeAliasType` for defining aliases on type unions.
They both work the same, but `mypy` doesn't like using variables in
types.

Additionally, I updated some READMEs and added a CI check to ensure that
the json in `specification/schema/` is always up to date.

.github/pre-commit

@@ -41,8 +41,8 @@ fi
 if ! poetry run mypy .
 then
     echo ""
-    echo "There are some python code style issues."
-    echo "Fix the warnings returned by `poetry run mypy .` first."
+    echo "There are some typing issues."
+    echo "Fix the warnings returned by 'poetry run mypy .' first."
     exit 1
 fi
 

.github/workflows/ci-py.yml

@@ -35,6 +35,7 @@ jobs:
           python:
             - 'quantinuum-hugr-py/**'
             - 'pyproject.toml'
+            - 'specification/schema/**'
 
   check:
     needs: changes
@@ -74,16 +75,54 @@ jobs:
       - name: Run tests
         run: poetry run pytest
 
+  # Ensure that the serialization schema is up to date
+  serialization-schema:
+    needs: [changes]
+    if: ${{ needs.changes.outputs.python == 'true' }}
+    name: Check serialization schema
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run sccache-cache
+        uses: mozilla-actions/[email protected]
+      - name: Install poetry
+        run: pipx install poetry
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+          cache: "poetry"
+      - name: Install the project libraries
+        run: poetry install
+      - name: Generate the updated schema
+        run: |
+          poetry run python scripts/generate_schema.py specification/schema/
+      - name: Check if the schema is up to date
+        run: |
+          git diff --exit-code --name-only specification/schema/
+          if [ $? -ne 0 ]; then
+            echo "The serialization schema is not up to date"
+            echo "Please run 'just update-schema' and commit the changes"
+            exit 1
+          fi
+
   # This is a meta job to mark successful completion of the required checks,
   # even if they are skipped due to no changes in the relevant files.
   required-checks:
     name: Required checks 🐍
-    needs: [changes, check]
-    if: always()
+    needs: [changes, check, serialization-schema]
+    if: ${{ !cancelled() }}
     runs-on: ubuntu-latest
     steps:
       - name: Fail if required checks failed
-        if: (failure() || cancelled())
+        # This condition should simply be `if: failure() || cancelled()`,
+        # but there seems to be a bug in the github workflow runner.
+        #
+        # See https://github.com/orgs/community/discussions/80788
+        if: |
+          needs.changes.result == 'failure' || needs.changes.result == 'cancelled' ||
+          needs.check.result == 'failure' || needs.check.result == 'cancelled' ||
+          needs.serialization-schema.result == 'failure' || needs.serialization-schema.result == 'cancelled'
         run: |
           echo "Required checks failed"
           echo "Please check the logs for more information"

.github/workflows/ci-rs.yml

@@ -134,11 +134,18 @@ jobs:
   required-checks:
     name: Required checks 🦀
     needs: [changes, check, tests-stable]
-    if: always()
+    if: ${{ !cancelled() }}
     runs-on: ubuntu-latest
     steps:
       - name: Fail if required checks failed
-        if: (failure() || cancelled())
+        # This condition should simply be `if: failure() || cancelled()`,
+        # but there seems to be a bug in the github workflow runner.
+        #
+        # See https://github.com/orgs/community/discussions/80788
+        if: |
+          needs.changes.result == 'failure' || needs.changes.result == 'cancelled' ||
+          needs.check.result == 'failure' || needs.check.result == 'cancelled' ||
+          needs.tests-stable.result == 'failure' || needs.tests-stable.result == 'cancelled'
         run: |
           echo "Required checks failed"
           echo "Please check the logs for more information"

DEVELOPMENT.md

@@ -29,6 +29,8 @@ shell by setting up [direnv](https://devenv.sh/automatic-shell-activation/).
 To setup the environment manually you will need:
 
 - Rust: https://www.rust-lang.org/tools/install
+- Just: https://just.systems/
+- Poetry: https://python-poetry.org/
 
 You can use the git hook in [`.github/pre-commit`](.github/pre-commit) to automatically run the test and check formatting before committing.
 To install it, run:
@@ -44,8 +46,10 @@ ln -s .github/pre-commit .git/hooks/pre-push
 To compile and test the rust code, run:
 
 ```bash
-cargo build
-cargo test
+# Rust tests
+just test # or `cargo test`
+# Python tests
+just pytest
 ```
 
 Run the benchmarks with:
@@ -62,6 +66,8 @@ stable available.
 cargo +nightly miri test
 ```
 
+Run `just` to see all available commands.
+
 ## 💅 Coding Style
 
 The rustfmt tool is used to enforce a consistent rust coding style. The CI will fail if the code is not formatted correctly.
@@ -70,13 +76,19 @@ To format your code, run:
 
 ```bash
 # Format rust code
-cargo fmt
+just format
+```
+
+We also use various linters to catch common mistakes and enforce best practices. To run these, use:
+
+```bash
+just check
 ```
 
-We also check for clippy warnings, which are a set of linting rules for rust. To run clippy, run:
+To quickly fix common issues, run:
 
 ```bash
-cargo clippy --all-targets
+just fix
 ```
 
 ## 📈 Code Coverage
@@ -85,9 +97,19 @@ We run coverage checks on the CI. Once you submit a PR, you can review the
 line-by-line coverage report on
 [codecov](https://app.codecov.io/gh/CQCL/hugr/commits?branch=All%20branches).
 
-To run the coverage checks locally, install `cargo-llvm-cov`, generate the report with:
+To run the coverage checks locally, first install `cargo-llvm-cov`.
+
+```bash
+cargo install cargo-llvm-cov
+```
+
+Then run the tests:
+
 ```bash
-cargo llvm-cov --lcov > lcov.info
+# Rust test coverage
+just coverage
+# Python test 
+just pycoverage
 ```
 
 and open it with your favourite coverage viewer. In VSCode, you can use

codecov.yml

@@ -15,6 +15,7 @@ coverage:
 # Ignore tests and binaries
 ignore:
   - "quantinuum-hugr-py/tests"
+  - "scripts"
 
 # Coverage groups config
 flag_management:

devenv.nix

@@ -30,6 +30,14 @@ in
     export LLVM_PROFDATA="${pkgs.llvmPackages_16.libllvm}/bin/llvm-profdata"
   '';
 
+  languages.python = {
+    enable = true;
+    poetry = {
+      enable = true;
+      activate.enable = true;
+    };
+  };
+
   # https://devenv.sh/languages/
   # https://devenv.sh/reference/options/#languagesrustversion
   languages.rust = {

justfile

@@ -35,3 +35,7 @@ pytest:
 # Generate a python test coverage report
 pycoverage:
     poetry run pytest --cov=./ --cov-report=html
+
+# Update the HUGR schema
+update-schema:
+    poetry run python scripts/generate_schema.py specification/schema/