From 5c3557e9b04c8364c4fbab367c91462d06d01d12 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 13:53:50 +0200 Subject: [PATCH 01/18] ci(py): add maturin workflow --- .github/workflows/maturin.yaml | 169 +++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 .github/workflows/maturin.yaml diff --git a/.github/workflows/maturin.yaml b/.github/workflows/maturin.yaml new file mode 100644 index 0000000..be7c696 --- /dev/null +++ b/.github/workflows/maturin.yaml @@ -0,0 +1,169 @@ +# This file is autogenerated by maturin v1.7.0 +# To update, run +# +# maturin generate-ci github -m pyfuzon/Cargo.toml +# +name: CI + +on: + push: + branches: + - main + - master + tags: + - '*' + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64 + - runner: ubuntu-latest + target: x86 + - runner: ubuntu-latest + target: aarch64 + - runner: ubuntu-latest + target: armv7 + - runner: ubuntu-latest + target: s390x + - runner: ubuntu-latest + target: ppc64le + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path pyfuzon/Cargo.toml + sccache: 'true' + manylinux: auto + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-linux-${{ matrix.platform.target }} + path: dist + + musllinux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64 + - runner: ubuntu-latest + target: x86 + - runner: ubuntu-latest + target: aarch64 + - runner: ubuntu-latest + target: armv7 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path pyfuzon/Cargo.toml + sccache: 'true' + manylinux: musllinux_1_2 + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-musllinux-${{ matrix.platform.target }} + path: dist + + windows: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: windows-latest + target: x64 + - runner: windows-latest + target: x86 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + architecture: ${{ matrix.platform.target }} + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path pyfuzon/Cargo.toml + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-windows-${{ matrix.platform.target }} + path: dist + + macos: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: macos-12 + target: x86_64 + - runner: macos-14 + target: aarch64 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path pyfuzon/Cargo.toml + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-macos-${{ matrix.platform.target }} + path: dist + + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist --manifest-path pyfuzon/Cargo.toml + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: wheels-sdist + path: dist + + release: + name: Release + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/')" + needs: [linux, musllinux, windows, macos, sdist] + steps: + - uses: actions/download-artifact@v4 + - name: Publish to PyPI + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + with: + command: upload + args: --non-interactive --skip-existing wheels-*/* From 1357d58230af320f321bf529299010c2ee9aab80 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 14:53:33 +0200 Subject: [PATCH 02/18] ci(py): drop windows build, as rff is unix-only --- .github/workflows/maturin.yaml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/.github/workflows/maturin.yaml b/.github/workflows/maturin.yaml index be7c696..3b7c370 100644 --- a/.github/workflows/maturin.yaml +++ b/.github/workflows/maturin.yaml @@ -85,33 +85,6 @@ jobs: name: wheels-musllinux-${{ matrix.platform.target }} path: dist - windows: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: windows-latest - target: x64 - - runner: windows-latest - target: x86 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.x - architecture: ${{ matrix.platform.target }} - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path pyfuzon/Cargo.toml - sccache: 'true' - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: wheels-windows-${{ matrix.platform.target }} - path: dist - macos: runs-on: ${{ matrix.platform.runner }} strategy: From 3c29ed5ba4b511bb13f844c9647dbf371b8f2759 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 15:06:07 +0200 Subject: [PATCH 03/18] ci(py): drop windows job dep --- .github/workflows/maturin.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maturin.yaml b/.github/workflows/maturin.yaml index 3b7c370..8bf94d0 100644 --- a/.github/workflows/maturin.yaml +++ b/.github/workflows/maturin.yaml @@ -130,7 +130,7 @@ jobs: name: Release runs-on: ubuntu-latest if: "startsWith(github.ref, 'refs/tags/')" - needs: [linux, musllinux, windows, macos, sdist] + needs: [linux, musllinux, macos, sdist] steps: - uses: actions/download-artifact@v4 - name: Publish to PyPI From 26ae574ec299958188f8c6ff03d2277344e936e9 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 15:11:48 +0200 Subject: [PATCH 04/18] docs: contrib guide --- CONTRIBUTING.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..199a490 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,37 @@ +# Contributing + +**Thank you very much for considering contributing to this project!** + +We welcome any form of contribution: + +- New Issues (feature requests, bug reports, questions, ideas, ...) +- Pull Requests (documentation improvements, code improvements, new features, + ...) + +**Note**: Before you take the time to open a pull request, please open an issue +first. This will give us the chance to discuss any potential changes first. + +## Opening a Pull Request + +Pull requests are the best way to propose changes to the codebase. We actively +welcome your pull requests: + +1. Fork the repo and create your branch from `main`. +2. If you've added code that should be tested, add tests. +3. If you've changed APIs, update the documentation. +4. Ensure the test suite passes: the CI will help you. +5. Create the pull request. + +**Note:** Pull-requests can only be merged if the CI passes. + +## License + +By contributing, you agree that your contributions will be licensed under this +[license](LICENSE). + +## Important Links + +- [Open Issues](https://github.com/sdsc-ordes/fuzon/issues) +- [Open Pull Requests](https://github.com/sdsc-ordes/fuzon/pulls) +- [Development Section in the README](https://github.com/sdsc-ordes/fuzon#development) +- [LICENSE](https://github.com/sdsc-ordes/fuzon/blob/main/LICENSE) From 8822485c7fa9a4916fee055d0c3037a7f53ea3bb Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 15:12:49 +0200 Subject: [PATCH 05/18] chore: generic config files --- .editorconfig | 17 +++++++ .gitattributes | 106 ++++++++++++++++++++++++++++++++++++++++++++ .prettierrc | 9 ++++ rust-toolchain.toml | 4 ++ rustfmt.toml | 3 ++ 5 files changed, 139 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitattributes create mode 100644 .prettierrc create mode 100644 rust-toolchain.toml create mode 100644 rustfmt.toml diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..004129e --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 4 + +[*.{json,yaml,yml}] +indent_size = 2 + +[*.nix] +indent_size = 2 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..11eaeb2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,106 @@ + +# Attribute file for assigning key=value pairs to paths. +# Any edit to this file needs to be reviewed by admins. +# These attributes define among other things +# which files are transferred into Git LFS when +# staging/commiting. +# +# Please read the docs first: https://git-scm.com/docs/gitattributes +# +# Restrictions: +# - No backslashes for paths! (also on Windows) +# - For spaces in paths use `[[:space:]]`. +# + +# Overall LF lineendings for the +# following files. +# =============================== +* text=auto eol=lf + +# Default attribute patterns for common binary files +# which should be stored in Git LFS. +# ================================================== +# Images +*.[Ii][Cc][Oo] filter=lfs diff=lfs merge=lfs -text +*.[Jj][Pp][Gg] filter=lfs diff=lfs merge=lfs -text +*.[Jj][Pp][Ee][Gg] filter=lfs diff=lfs merge=lfs -text +*.[Pp][Nn][Gg] filter=lfs diff=lfs merge=lfs -text +*.[Gg][Ii][Ff] filter=lfs diff=lfs merge=lfs -text +*.[Tt][Ii][Ff] filter=lfs diff=lfs merge=lfs -text +*.[Tt][Ii][Ff][Ff] filter=lfs diff=lfs merge=lfs -text +*.[Pp][Ss][Dd] filter=lfs diff=lfs merge=lfs -text +*.[Aa][Ii] filter=lfs diff=lfs merge=lfs -text + +# Media +*.[Pp][Dd][Ff] filter=lfs diff=lfs merge=lfs -text + +# Video/Audio +*.[Mm][Oo][Vv] filter=lfs diff=lfs merge=lfs -text +*.[Mm][Pp]4 filter=lfs diff=lfs merge=lfs -text +*.[Mm][Pp]2 filter=lfs diff=lfs merge=lfs -text +*.[Mm]4[Vv] filter=lfs diff=lfs merge=lfs -text +*.[Ww][Mm][Vv] filter=lfs diff=lfs merge=lfs -text +*.[Ww][Mm][Vv] filter=lfs diff=lfs merge=lfs -text +*.[Mm][Pp][Ee] filter=lfs diff=lfs merge=lfs -text +*.[Mm][Pp][Gg] filter=lfs diff=lfs merge=lfs -text +*.[Mm][Pp][Ee][Gg] filter=lfs diff=lfs merge=lfs -text +*.[Aa][Vv][Ii] filter=lfs diff=lfs merge=lfs -text +*.[Ww][Ee][Bb][Mm] filter=lfs diff=lfs merge=lfs -text +*.[Oo][Gg][Gg] filter=lfs diff=lfs merge=lfs -text +*.[Mm][Pp]3 filter=lfs diff=lfs merge=lfs -text +*.[Ww][Aa][Vv] filter=lfs diff=lfs merge=lfs -text + +# MS Office +*.[Dd][Oo][Cc] filter=lfs diff=lfs merge=lfs -text +*.[Dd][Oo][Cc][Xx] filter=lfs diff=lfs merge=lfs -text +*.[Pp][Pp][Tt] filter=lfs diff=lfs merge=lfs -text +*.[Pp][Pp][Tt][Xx] filter=lfs diff=lfs merge=lfs -text +*.[Xx][Ll][Ss] filter=lfs diff=lfs merge=lfs -text +*.[Xx][Ll][Ss][Xx] filter=lfs diff=lfs merge=lfs -text +*.[Dd][Oo][Tt][Xx] filter=lfs diff=lfs merge=lfs -text + +# Fonts +*.[Tt][Tt][Ff] filter=lfs diff=lfs merge=lfs -text +*.[Oo][Tt][Ff] filter=lfs diff=lfs merge=lfs -text +*.[Ww][Oo][Ff][Ff] filter=lfs diff=lfs merge=lfs -text +*.[Ww][Oo][Ff][Ff]2 filter=lfs diff=lfs merge=lfs -text + +# Data Zips +*.[Zz][Ii][Pp] filter=lfs diff=lfs merge=lfs -text +*.[Zz] filter=lfs diff=lfs merge=lfs -text +*.[Gg][Zz] filter=lfs diff=lfs merge=lfs -text +*.[Tt][Aa][Rr] filter=lfs diff=lfs merge=lfs -text +*.[Tt][Gg][Zz] filter=lfs diff=lfs merge=lfs -text +*.[Zz][Ii][Pp] filter=lfs diff=lfs merge=lfs -text +*.[Bb][Zz]2 filter=lfs diff=lfs merge=lfs -text +*.[Ll][Zz][Mm][Aa] filter=lfs diff=lfs merge=lfs -text +*.[Ll][Zz] filter=lfs diff=lfs merge=lfs -text +*.[Ll][Zz]4 filter=lfs diff=lfs merge=lfs -text +*.7[Zz] filter=lfs diff=lfs merge=lfs -text +*.[Ss]7[Zz] filter=lfs diff=lfs merge=lfs -text +*.[Rr][Aa][Rr] filter=lfs diff=lfs merge=lfs -text +*.[Bb][Zz][Ii][Pp] filter=lfs diff=lfs merge=lfs -text + +# Packages +*.[Xx][Pp][Ii] filter=lfs diff=lfs merge=lfs -text +*.[Dd][Mm][Gg] filter=lfs diff=lfs merge=lfs -text +*.[Ii][Mm][Gg] filter=lfs diff=lfs merge=lfs -text +*.[Ii][Ss][Oo] filter=lfs diff=lfs merge=lfs -text +*.[Ww][Ii][Mm] filter=lfs diff=lfs merge=lfs -text +*.[Ss][Ww][Mm] filter=lfs diff=lfs merge=lfs -text +*.[Ee][Ss][Dd] filter=lfs diff=lfs merge=lfs -text +*.[Mm][Ss][Ii] filter=lfs diff=lfs merge=lfs -text +*.[Pp][Kk][Gg] filter=lfs diff=lfs merge=lfs -text +*.[Mm][Pp][Kk][Gg] filter=lfs diff=lfs merge=lfs -text +*.[Dd][Ee][Bb] filter=lfs diff=lfs merge=lfs -text +*.[Rr][Pp][Mm] filter=lfs diff=lfs merge=lfs -text +*.[Jj][Aa][Rr] filter=lfs diff=lfs merge=lfs -text + +# Libraries +*.[Pp][Dd][Bb] filter=lfs diff=lfs merge=lfs -text +*.[Dd][Ll][Ll] filter=lfs diff=lfs merge=lfs -text +*.[Ll][Ii][Bb] filter=lfs diff=lfs merge=lfs -text + +# General binary +*.[Bb][Ii][Nn] filter=lfs diff=lfs merge=lfs -text +*.[Ee][Xx][Ee] filter=lfs diff=lfs merge=lfs -text diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..e480fd5 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,9 @@ +{ + "printWidth": 80, + "tabWidth": 2, + "semi": false, + "singleQuote": false, + "overrides": [ + { "files": "*.md", "options": { "proseWrap": "always", "tabWidth": 2 } } + ] +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..782c141 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "nightly-2024-06-17" +components = [ "rustfmt", "rust-analyzer", "miri", "rust-docs", "clippy", "rust-src"] +profile = "default" diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..9aeb106 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,3 @@ +edition = "2021" +reorder_imports = true +imports_granularity = "Crate" From cc817ea45739ea6f5a3fb6654ed95178d8eafadf Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 16:08:32 +0200 Subject: [PATCH 06/18] chore: add nix-based tooling --- .envrc | 1 + tools/bench/benchmark.sh | 171 ++++++++++++++++++++++++++++++ tools/build-image.sh | 68 ++++++++++++ tools/build-package.sh | 19 ++++ tools/ci/assert-tag.sh | 67 ++++++++++++ tools/ci/create-github-release.sh | 27 +++++ tools/ci/images/Containerfile | 43 ++++++++ tools/ci/setup-git.sh | 15 +++ tools/ci/upload-ci-images.sh | 41 +++++++ tools/ci/upload-image.sh | 52 +++++++++ tools/format-rust.sh | 22 ++++ tools/general.sh | 108 +++++++++++++++++++ tools/lint-rust.sh | 22 ++++ tools/lint-ub-rust.sh | 18 ++++ tools/nix/flake.nix | 144 +++++++++++++++++++++++++ tools/nix/images/ci.nix | 22 ++++ tools/nix/images/fuzon.nix | 30 ++++++ tools/nix/pkgs/fuzon.nix | 32 ++++++ tools/release.sh | 132 +++++++++++++++++++++++ 19 files changed, 1034 insertions(+) create mode 100644 .envrc create mode 100644 tools/bench/benchmark.sh create mode 100755 tools/build-image.sh create mode 100755 tools/build-package.sh create mode 100755 tools/ci/assert-tag.sh create mode 100755 tools/ci/create-github-release.sh create mode 100644 tools/ci/images/Containerfile create mode 100755 tools/ci/setup-git.sh create mode 100755 tools/ci/upload-ci-images.sh create mode 100755 tools/ci/upload-image.sh create mode 100755 tools/format-rust.sh create mode 100755 tools/general.sh create mode 100755 tools/lint-rust.sh create mode 100755 tools/lint-ub-rust.sh create mode 100644 tools/nix/flake.nix create mode 100644 tools/nix/images/ci.nix create mode 100644 tools/nix/images/fuzon.nix create mode 100644 tools/nix/pkgs/fuzon.nix create mode 100755 tools/release.sh diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..b4232c4 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake ./tools/nix#default diff --git a/tools/bench/benchmark.sh b/tools/bench/benchmark.sh new file mode 100644 index 0000000..30db925 --- /dev/null +++ b/tools/bench/benchmark.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash + +# Benchmark runtime and memory usage of tripsu +# Compares the working directory version against a baseline branch (main by default) + +set -euo pipefail + +### Final output path +OUTPUT="profiling.md" +PROFILE='release' +BUILD_ARGS=( ) +[[ "${PROFILE}" == 'release' ]] && BUILD_ARGS+=( '--release' ) +### setup binaries + +# baseline binary +BASE_BRANCH='main' + +BASE_DIR=$(mktemp -d) +BASE_URL="$(git config --get remote.origin.url)" +( + GIT_CLONE_PROTECTION_ACTIVE=false \ + git clone \ + --branch "${BASE_BRANCH}" \ + "${BASE_URL}" \ + "${BASE_DIR}" \ + && cd "${BASE_DIR}" \ + && just build "${BUILD_ARGS[@]}" +) +BASE_BIN="${BASE_DIR}/target/${PROFILE}/tripsu" + +# current binary +COMP_BRANCH="$(git rev-parse --abbrev-ref HEAD)" +just build "${BUILD_ARGS[@]}" +COMP_BIN="./target/${PROFILE}/tripsu" + +# setup data +DATA_URL="https://ftp.uniprot.org/pub/databases/uniprot/current_release/rdf/proteomes.rdf.xz" +INPUT="/tmp/proteomes.nt" + +# Download data if needed +if [ ! -f ${INPUT} ]; then + curl "${DATA_URL}" \ + | xz -dc - \ + | rdfpipe-rs -i rdf-xml -o nt - \ + > "${INPUT}" || rm "${INPUT}" +fi + +# setup config +RULES=$(mktemp) +BASE_IDX=$(mktemp) +COMP_IDX=$(mktemp) + +cat << EOF > "${RULES}" + +nodes: + of_type: + - "http://purl.uniprot.org/core/Proteome" + - "http://purl.uniprot.org/core/Strain" + +objects: + on_type_predicate: + "http://purl.uniprot.org/core/Submission_Citation": + - "http://purl.uniprot.org/core/author" + + on_predicate: + - "http://purl.org/dc/terms/identifier" + +EOF + +### Commands to benchmark +BASE_CMD_IDX="${BASE_BIN} index -o ${BASE_IDX} ${INPUT}" +COMP_CMD_IDX="${COMP_BIN} index -o ${COMP_IDX} ${INPUT}" +BASE_CMD_PSD="${BASE_BIN} pseudo -r ${RULES} -x ${BASE_IDX} ${INPUT}" +COMP_CMD_PSD="${COMP_BIN} pseudo -r ${RULES} -x ${COMP_IDX} ${INPUT}" + +### functions for profiling + +cpu_prof() { + local branch1=$1 + local cmd1=$2 + local branch2=$3 + local cmd2=$4 + local out=$5 + hyperfine --export-markdown "${out}" -r 5 \ + -n "${branch1}" "${cmd1}" \ + -n "${branch2}" "${cmd2}" +} + +mem_prof() { + local name=$1 + local cmd=$2 + local heap_out + heap_out=$(mktemp) + echo -n "$name: " + # shellcheck disable=SC2086 + heaptrack -o "${heap_out}" ${cmd} >/dev/null + heaptrack_print "${heap_out}.zst" \ + | grep '^peak heap memory' +} + +make_report() { + local cpu_index=$1 + local cpu_pseudo=$2 + local mem_index=$3 + local mem_pseudo=$4 + local base_branch=$5 + + cat <<-MD + # tripsu profiling + + > date: $(date -u +%Y-%m-%d) + + Comparing $(git branch --show-current) against $base_branch. + + ## Timings + + Run time compared using hyperfine + + ### Indexing + + $(cat "${cpu_index}") + + ### Pseudonymization + + $(cat "${cpu_pseudo}") + + ## Memory + + Heap memory usage compared using heaptrack + + ### Indexing + + $(cat "${mem_index}") + + ### Pseudonymization + + $(cat "${mem_pseudo}") + MD +} + + +### Run profiling + +## Profile cpu time +HYPF_IDX_OUT=$(mktemp) +HYPF_PSD_OUT=$(mktemp) + +# indexing +cpu_prof "${BASE_BRANCH}" "${BASE_CMD_IDX}" \ + "${COMP_BRANCH}" "${COMP_CMD_IDX}" "${HYPF_IDX_OUT}" +# pseudonymization +cpu_prof "${BASE_BRANCH}" "${BASE_CMD_IDX}" \ + "${COMP_BRANCH}" "${COMP_CMD_IDX}" "${HYPF_PSD_OUT}" + +## Profile memory +HEAP_IDX_OUT=$(mktemp) +HEAP_PSD_OUT=$(mktemp) + +# indexing +mem_prof "${BASE_BRANCH}" "${BASE_CMD_IDX}" > "${HEAP_IDX_OUT}" +mem_prof "${COMP_BRANCH}" "${COMP_CMD_IDX}" >> "${HEAP_IDX_OUT}" +# pseudonymization +mem_prof "${BASE_BRANCH}" "${BASE_CMD_PSD}" > "${HEAP_PSD_OUT}" +mem_prof "${COMP_BRANCH}" "${COMP_CMD_PSD}" >> "${HEAP_PSD_OUT}" + + +### Reporting +make_report \ + "${HYPF_IDX_OUT}" "${HYPF_PSD_OUT}" \ + "${HEAP_IDX_OUT}" "${HEAP_PSD_OUT}" \ + "${BASE_BRANCH}" > "${OUTPUT}" diff --git a/tools/build-image.sh b/tools/build-image.sh new file mode 100755 index 0000000..11459c7 --- /dev/null +++ b/tools/build-image.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +# +# Build the Nix container image. + +set -euo pipefail + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +VERSION_FILE="$ROOT_DIR/Cargo.toml" + +function clean_up() { + if ! ci_is_running; then + # Never change the version file, only + # on explicit `just release`. + print_info "Restoring '$VERSION_FILE'." + git restore "$VERSION_FILE" || true + fi +} + +trap clean_up EXIT + +function main() { + args=("$@") + + if ! ci_is_running || ! ci_is_release; then + print_info "Building image for development." + + # Define the image version from Git SHA + version="0.0.0-dev.$(git rev-parse --short=7 HEAD)" + + # Write the temporary version file (gets restored...) + dasel put -r toml -f "$VERSION_FILE" -t string -v "$version" .package.version + else + print_info "Building image for release." + + # When CI and in Release, the requested version must match. + version=$(dasel -r toml -f "$VERSION_FILE" .package.version -w yaml) + + release_version=${GITHUB_REF##*prepare-v} + + if [ "$version" != "$release_version" ]; then + die "The version '$version' in '$VERSION_FILE' does not corresponds" \ + "with the version '$release_version' to build." \ + "Update the version file to align!" \ + "Nix is pure and cannot rely on Git tags to" \ + "get the version from." + fi + fi + + image_name=$(nix eval --raw "./tools/nix#images.tripsu.imageName") + image_tag=$(nix eval --raw "./tools/nix#images.tripsu.imageTag") + + # We cannot use `:` before the tag, because `skopeo` + # is unable to read this path correctly, because it + # stupidly deals with it. + dir="build/image/$image_name|$image_tag.tar.gz" + + cd "$ROOT_DIR" + + print_info "Building image '$dir'." + nix --version + nix build -L "./tools/nix#images.tripsu" \ + --out-link "$dir" "${args[@]}" +} + +main "$@" diff --git a/tools/build-package.sh b/tools/build-package.sh new file mode 100755 index 0000000..3d9d1c1 --- /dev/null +++ b/tools/build-package.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +# +# Build the Nix container image. + +set -euo pipefail + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +cd "$ROOT_DIR" + +dir="build/package" + +print_info "Building the package." +nix --version +nix build -L "./tools/nix#tripsu" \ + --out-link "$dir" \ + "$@" diff --git a/tools/ci/assert-tag.sh b/tools/ci/assert-tag.sh new file mode 100755 index 0000000..16c8872 --- /dev/null +++ b/tools/ci/assert-tag.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +# +# Assert that the release tag exists +# and check that its on main. +# On `--push` do push the tag. + +set -euo pipefail + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +cd "$ROOT_DIR" + +RELEASE_BRANCH="main" + +function main() { + local type="$1" + local prepare_tag="$2" + + local release_tag=${prepare_tag##*prepare-} + + if [ "$type" = "push" ]; then + print_info "Pushing tag '$release_tag'." + + git push origin "$release_tag" || + die "Could not push tag." + + elif [ "$type" = "create-and-check" ]; then + + print_info "Create tag '$release_tag' and check." + + # Gets the message on the annotated commit: + deref() { + git for-each-ref "refs/tags/$release_tag" --format="%($1)" + } + + deref contents + + # Creates a new tag with the same message, + # including trailing headers. + git tag -a -m "$(deref contents)" "$release_tag" || + die "Could not create tag." + + # Fetch the branch. + git fetch --depth 50 origin "$RELEASE_BRANCH" + + # Check if its reachable. + if [ -z "$(git rev-list --first-parent \ + --ancestry-path \ + "$release_tag^..origin/$RELEASE_BRANCH")" ]; then + git log --oneline --graph "$RELEASE_BRANCH" "$release_tag" || true + die "Tag is not reachable from '$RELEASE_BRANCH' (--first-parent) !" + fi + + elif [ "$type" = "cleanup" ]; then + + print_info "Cleanup the prepare tag." + git tag -d "$prepare_tag" || true + git push origin :"$prepare_tag" || true + + else + die "Type '$type' is not known." + fi +} + +main "$@" diff --git a/tools/ci/create-github-release.sh b/tools/ci/create-github-release.sh new file mode 100755 index 0000000..2c0102c --- /dev/null +++ b/tools/ci/create-github-release.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +# This script is sourced in each step. +set -u +set -e + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +function main() { + local prepare_tag="$1" + local repo="$2" + + local tag=${prepare_tag#prepare-} + local version=${tag#v} + + print_info "Creating Github release ... " + + gh release create "$tag" \ + --repo="$repo" \ + --title="tripsu: $version" \ + --generate-notes + + print_info "Successfully created release. All done." +} + +main "$@" diff --git a/tools/ci/images/Containerfile b/tools/ci/images/Containerfile new file mode 100644 index 0000000..27c23cf --- /dev/null +++ b/tools/ci/images/Containerfile @@ -0,0 +1,43 @@ +# This is a docker image containing docker and a Nix store. +# This enables to either run Docker images inside this one, +# or use `nix develop` to start a sandboxed environment to +# do other non-docker related stuff. + +FROM alpine:latest as base-podman +LABEL org.opencontainers.image.source https://github.com/sdsc-ordes/tripsu +LABEL org.opencontainers.image.description "CI container image for tripsu" +LABEL org.opencontainers.image.license "Apache" + +RUN apk add findutils coreutils git jq curl bash just parallel podman + +# Nix Image +# =============================================== +FROM base-podman as ci-nix +ARG USER_NAME=ci +ARG USER_UID=1000 +ARG USER_GID=1000 +ARG CACHE_TOOLCHAIN=false +ARG USER_HOME="/github/home" + +RUN [ "TARGETPLATFORM" = "linux/amd64" ] || echo "Platform not yet supported." +COPY ./tools /container-setup/tools + +# Install Nix and pre-cache the env. +RUN bash -c ". /container-setup/tools/general.sh && ci_setup_nix" + +# Create user (does not work because action/checkout uses `root` in its own container ... 💩) +# Leave this code here for a reference: +# RUN mkdir -p "$USER_HOME" && \ +# adduser "$USER_NAME" -s /bin/zsh -D -u "$USER_UID" -g "$USER_GID" -h "$USER_HOME/$USER_NAME" && \ +# mkdir -p /etc/sudoers.d && \ +# echo "$USER_NAME ALL=(root) NOPASSWD:ALL" > "/etc/sudoers.d/$USER_NAME" && \ +# chmod 0440 "/etc/sudoers.d/$USER_NAME" && \ +# chown -R "$USER_NAME:$USER_NAME" /home /container-setup +# USER "$USER_NAME" + +COPY rust-toolchain.toml /container-setup/ +RUN [ "${CACHE_TOOLCHAIN}" = "false" ] || { cd /container-setup && \ + git init && git add . && \ + nix --accept-flake-config \ + build --no-link "./tools/nix#devShells.x86_64-linux.ci" && \ + nix store optimise; } diff --git a/tools/ci/setup-git.sh b/tools/ci/setup-git.sh new file mode 100755 index 0000000..2b325c2 --- /dev/null +++ b/tools/ci/setup-git.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +# This script is sourced in each step. +set -u +set -e + +git config --global safe.directory "*" || { + echo "Could not overwrite safe.directory in Git config." >&2 + exit 1 +} + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +ci_setup_git diff --git a/tools/ci/upload-ci-images.sh b/tools/ci/upload-ci-images.sh new file mode 100755 index 0000000..d67d092 --- /dev/null +++ b/tools/ci/upload-ci-images.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +set -e +set -u + +DIR=$(cd -- "$(dirname -- "$0")" &>/dev/null && pwd) +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +cd "$ROOT_DIR" + +function build_ci_image() { + local image_type="$1" + local repository="$2" + local version="$3" + local tag="$image_type-$version" + + local image_name="$repository:$tag" + + print_info "Building image '$image_name'." + + ci_container_mgr build -f "$container_file" \ + --target "$image_type" \ + -t "$image_name" \ + . || die "Could not build image." + + ci_container_mgr push "$image_name" || die "Could not upload image." +} + +tag="${1:-1.0.0}" +repository="${2:-ghcr.io/sdsc-ordes/tripsu}" +container_file="$DIR/images/Containerfile" + +if [ "${CI:-}" = "true" ]; then + ci_container_mgr_login "$DOCKER_REPOSITORY_READ_USERNAME" "$DOCKER_REPOSITORY_READ_TOKEN" +fi + +readarray -t images < <(grep -E "as ci-.*" "$container_file" | sed -E 's@.*as (ci-.*)$@\1@g') +for image in "${images[@]}"; do + build_ci_image "$image" "$repository" "$tag" +done diff --git a/tools/ci/upload-image.sh b/tools/ci/upload-image.sh new file mode 100755 index 0000000..1be4d40 --- /dev/null +++ b/tools/ci/upload-image.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +# +# Push image to the registry. + +set -euo pipefail + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +cd "$ROOT_DIR" + +function main() { + + local username=${REGISTRY_USERNAME:-$USERNAME} + local password=${REGISTRY_PASSWORD:-$PASSWORD} + + if [ -z "$username" ] || [ -z "$password" ]; then + die "'USERNAME' or 'PASSWORD' env. variables not set." + fi + + if ! ci_is_running; then + die "This script should only be executed in CI" + fi + + local image_names=() + readarray -t image_names < <(cd build/image && find ./ -name "*.tar.gz") + + for image_name in "${image_names[@]}"; do + + image_path="$ROOT_DIR/build/image/$image_name" + image_name=${image_name%.tar.gz} # Split `.tar.gz` from end. + image_name=${image_name#./} # Split `./` from front. + image_name=${image_name/|/:} # Replace `|` with `:`. + + print_info "Uploading image: '$image_name' in '$image_path'." + + print_info "Read the image from file '$image_path' and " \ + "directly push to registry '$image_name'." + + skopeo \ + --insecure-policy \ + copy \ + --dest-username "$(cat <(echo "$username"))" \ + --dest-password "$(cat <(echo "$password"))" \ + "docker-archive://$image_path" \ + "docker://$image_name" + + done +} + +main "$@" diff --git a/tools/format-rust.sh b/tools/format-rust.sh new file mode 100755 index 0000000..c2a8d8f --- /dev/null +++ b/tools/format-rust.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +set -e +set -u + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +cd "$ROOT_DIR" + +fmt_args=() +if ci_is_running; then + fmt_args+=("--check") +fi + +cargo --version +cargo fmt --version + +print_info "Run Rust format." +cargo fmt "${fmt_args[@]}" "$@" + +print_info "Done." diff --git a/tools/general.sh b/tools/general.sh new file mode 100755 index 0000000..068bcd7 --- /dev/null +++ b/tools/general.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +# shellcheck disable=SC2154,SC2086 + +function _print() { + local color="$1" + local flags="$2" + local header="$3" + shift 3 + + local hasColor="0" + if [ "${FORCE_COLOR:-}" != 1 ]; then + [ -t 1 ] && hasColor="1" + else + hasColor="1" + fi + + if [ "$hasColor" = "0" ] || [ "${LOG_COLORS:-}" = "false" ]; then + local msg + msg=$(printf '%b\n' "$@") + msg="${msg//$'\n'/$'\n' }" + echo $flags -e "-- $header$msg" + else + local s=$'\033' e='[0m' + local msg + msg=$(printf "%b\n" "$@") + msg="${msg//$'\n'/$'\n' }" + echo $flags -e "${s}${color}-- $header$msg${s}${e}" + fi +} +function print_info() { + _print "[0;94m" "" "" "$@" +} + +function print_warning() { + _print "[0;31m" "" "WARN: " "$@" >&2 +} + +function print_error() { + _print "[0;31m" "" "ERROR: " "$@" >&2 +} + +function die() { + print_error "$@" + exit 1 +} + +function ci_is_running() { + if [ "${CI:-}" = "true" ]; then + return 0 + fi + + return 1 +} + +function ci_is_release() { + if [ "${CI_IS_RELEASE:-}" = "true" ]; then + return 0 + fi + + return 1 +} + +function ci_setup_git() { + git config --global user.name "SDSC CI" + git config --global user.email "ci@sdsc.ethz.ch" +} + +function ci_setup_nix() { + local install_prefix="${1:-/usr/sbin}" + + print_info "Install Nix." + apk add curl bash xz shadow + sh <(curl -L https://nixos.org/nix/install) --daemon --yes + cp /root/.nix-profile/bin/* "$install_prefix/" + + print_info "Enable Features for Nix." + mkdir -p ~/.config/nix + { + echo "experimental-features = nix-command flakes" + echo "accept-flake-config = true" + } >~/.config/nix/nix.conf +} + +# Run the container manager which is defined. +# in env. variable `CONTAINER_MGR` +# (by default `podman` if existing). +function ci_container_mgr() { + local mgr="${CONTAINER_MGR:-podman}" + + if command -v "$mgr" &>/dev/null; then + echo -e "Running '$mgr' as:\n$(printf "'%s' " "podman" "$@")" >&2 + "$mgr" "$@" + else + echo -e "Running docker as:\n$(printf "'%s' " "docker" "$@")" + docker "$@" + fi +} + +function ci_container_mgr_login() { + local user="$1" + local token="$2" + + [ -n "$token" ] || die "Docker login token is empty" + echo "$token" | + ci_container_mgr login --password-stdin --username "$user" || + die "Could not log into docker." +} diff --git a/tools/lint-rust.sh b/tools/lint-rust.sh new file mode 100755 index 0000000..848c4d8 --- /dev/null +++ b/tools/lint-rust.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +set -e +set -u + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +cd "$ROOT_DIR" + +cargo --version +cargo clippy --version + +print_info "Run Rust Clippy linter." + +cargo clippy --no-deps -- -D warnings -A clippy::needless_return "$@" || + { + git diff --name-status || true + die "Rust clippy failed." + } + +print_info "Done." diff --git a/tools/lint-ub-rust.sh b/tools/lint-ub-rust.sh new file mode 100755 index 0000000..71c1725 --- /dev/null +++ b/tools/lint-ub-rust.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +set -e +set -u + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +cd "$ROOT_DIR" + +cargo --version +cargo miri --version + +print_info "Run Rust Miri to check undefined behaviour." +cargo miri test "$@" || + die "Rust Miri failed." + +print_info "Done." diff --git a/tools/nix/flake.nix b/tools/nix/flake.nix new file mode 100644 index 0000000..7370a33 --- /dev/null +++ b/tools/nix/flake.nix @@ -0,0 +1,144 @@ +{ + description = "fuzon"; + + nixConfig = { + substituters = [ + # Add here some other mirror if needed. + "https://cache.nixos.org/" + ]; + extra-substituters = [ + # Nix community's cache server + "https://nix-community.cachix.org" + ]; + extra-trusted-public-keys = [ + "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + ]; + }; + + inputs = { + # Nixpkgs + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + + # You can access packages and modules from different nixpkgs revs + # at the same time. Here's an working example: + nixpkgsStable.url = "github:nixos/nixpkgs/nixos-23.11"; + # Also see the 'stable-packages' overlay at 'overlays/default.nix'. + + flake-utils.url = "github:numtide/flake-utils"; + + # The Rust overlay to include the latest toolchain. + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs = { + nixpkgs.follows = "nixpkgs"; + }; + }; + }; + + outputs = { + nixpkgs, + flake-utils, + rust-overlay, + ... + }: let + # This is string (without toString it would be a `path` which is put into the store) + rootDir = toString ./. + "../../.."; + python = pkgs.python312; + in + flake-utils.lib.eachDefaultSystem + # Creates an attribute map `{ devShells..default = ...}` + # by calling this function: + ( + system: let + overlays = [(import rust-overlay)]; + + # Import nixpkgs and load it into pkgs. + # Overlay the rust toolchain + lib = nixpkgs.lib; + pkgs = import nixpkgs { + inherit system overlays; + }; + + # Set the rust toolchain from the `rust-toolchain.toml`. + rustToolchain = pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ../../rust-toolchain.toml; + + # Things needed only at compile-time. + nativeBuildInputsBasic = with pkgs; [ + maturin + findutils + coreutils + bash + zsh + curl + git + jq + ]; + + # Things needed only at compile-time. + nativeBuildInputsDev = with pkgs; [ + rustToolchain + cargo-watch + just + + skopeo + dasel + ]; + + benchInputs = with pkgs; [ + hyperfine + heaptrack + ]; + + # Things needed at runtime. + buildInputs = []; + + # The package of this CLI tool. + # The global version for tripsu. + # This is gonna get tooled later. + fuzon = (import ./pkgs/fuzon.nix) { + inherit rootDir rustToolchain pkgs lib; + }; + in + with pkgs; rec { + devShells = { + default = mkShell { + inherit buildInputs; + nativeBuildInputs = nativeBuildInputsBasic ++ nativeBuildInputsDev; + }; + bench = mkShell { + inherit buildInputs; + nativeBuildInputs = nativeBuildInputsBasic + ++ nativeBuildInputsDev + ++ benchInputs; + }; + + ci = mkShell { + inherit buildInputs; + nativeBuildInputs = nativeBuildInputsBasic ++ nativeBuildInputsDev; + + # Due to some weird handling of TMPDIR inside containers: + # https://github.com/NixOS/nix/issues/8355 + # We have to reset the TMPDIR to make `nix build` work inside + # a development shell. + # Without `nix develop` it works. + shellHook = "unset TMPDIR"; + }; + }; + + packages = { + tripsu = fuzon; + + images = { + ci = (import ./images/ci.nix) { + inherit pkgs; + devShellDrv = devShells.default; + }; + + fuzon = (import ./images/fuzon.nix) { + inherit pkgs fuzon; + }; + }; + }; + } + ); +} diff --git a/tools/nix/images/ci.nix b/tools/nix/images/ci.nix new file mode 100644 index 0000000..ced142d --- /dev/null +++ b/tools/nix/images/ci.nix @@ -0,0 +1,22 @@ +{ + pkgs, + devShellDrv, + ... +}: let + version = "1.0.0"; # The version of these CI images. + image_name = "ghcr.io/sdsc-ordes/fuzon"; + + buildImage = type: + pkgs.dockerTools.buildNixShellImage { + name = image_name; + tag = "ci-${type}-${version}"; + drv = devShellDrv; + }; +in rec { + format = buildImage "format"; + lint = buildImage "lint"; + build = buildImage "build"; + test = buildImage "test"; + package = buildImage "package"; + deploy = buildImage "deploy"; +} diff --git a/tools/nix/images/fuzon.nix b/tools/nix/images/fuzon.nix new file mode 100644 index 0000000..32ba15e --- /dev/null +++ b/tools/nix/images/fuzon.nix @@ -0,0 +1,30 @@ +{ + pkgs, + fuzon, +}: +pkgs.dockerTools.buildLayeredImage { + name = "ghcr.io/sdsc-ordes/fuzon"; + tag = fuzon.version; + + contents = [fuzon]; + + fakeRootCommands = '' + ${pkgs.dockerTools.shadowSetup} + groupadd -r non-root + useradd -r -g non-root non-root + mkdir -p /workspace + chown non-root:non-root /workspace + ''; + enableFakechroot = true; + + config = { + Entrypoint = ["fuzon"]; + WorkingDir = "/workspace"; + Labels = { + "org.opencontainers.image.source" = "https://github.com/sdsc-ordes/fuzon"; + "org.opencontainers.image.description" = fuzon.meta.description; + "org.opencontainers.image.license" = "Apache-2.0"; + }; + User = "non-root"; + }; +} diff --git a/tools/nix/pkgs/fuzon.nix b/tools/nix/pkgs/fuzon.nix new file mode 100644 index 0000000..29dcfd0 --- /dev/null +++ b/tools/nix/pkgs/fuzon.nix @@ -0,0 +1,32 @@ +{ + pkgs, + lib, + rustToolchain, + rootDir, + ... +}: let + rustPlatform = pkgs.makeRustPlatform { + cargo = rustToolchain; + rustc = rustToolchain; + }; + + cargoFile = /. + rootDir + "fuzon/Cargo.toml"; + lockFile = /. + rootDir + "fuzon/Cargo.lock"; +in + rustPlatform.buildRustPackage { + name = "fuzon"; + src = /. + rootDir + "fuzon/"; + + version = (lib.importTOML cargoFile).package.version; + + cargoLock = { + inherit lockFile; + }; + + meta = { + description = "A CLI tool to fuzzy search ontology terms by their labels."; + homepage = "https://github.com/sdsc-ordes/fuzon"; + license = lib.licenses.asl20; + maintainers = ["gabyx" "cmdoret"]; + }; + } diff --git a/tools/release.sh b/tools/release.sh new file mode 100755 index 0000000..ba3ef13 --- /dev/null +++ b/tools/release.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +# +# Creating a prepare tag to trigger the release process on the +# Github workflow. Can only be called on `main`. +# +# Usage: release.sh "1.2.0" + +set -euo pipefail + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +RELEASE_BRANCH="${RELEASE_BRANCH:-main}" +VERSION_FILE="$ROOT_DIR/Cargo.toml" + +function delete_prepare_tags() { + readarray -t prepareTag < <(git tag --list "prepare-*") + + for tag in "${prepareTag[@]}"; do + print_info "Deleting prepare tag '$tag'." + git push -f origin ":${tag}" || true + git tag -d "$tag" || die + done +} + +function create_prepare_tag() { + local tag="$1" + + print_info "Tagging with '$tag'." + git tag -a -m "Version $version" "$tag" || die "Could not create tag." + + print_info "Tag contains:" + git cat-file -p "$tag" || die "Could not show tag content." + + print_info "Successfully created prepate tag '$tag'." +} + +function commit_version_file() { + local version="$1" + print_info "Writing new version file... (for Nix)" + + dasel put -r toml -f "$VERSION_FILE" -t string -v "$version" .package.version + + if ! git diff --exit-code --quiet; then + # Commit if we have change. + git add "$VERSION_FILE" + git commit -m "chore: release '$version'" + fi +} + +function trigger_build() { + local branch="$1" + local tag="$2" + + printf "Do you want to trigger the build? [y|n]: " + read -r answer + if [ "$answer" != "y" ]; then + die "Do not trigger build -> abort." + fi + + print_info "Pushing tag '$tag'." + git push -f origin --no-follow-tags "$branch" "$tag" + + print_info "Successfully triggered build." +} + +function check_new_version() { + local new_version="$1" + + # Check that is a version. + if [ "$(ci_container_mgr run --rm docker.io/alpine/semver semver "$new_version" | tail -1)" != "$new_version" ]; then + die "Your version '$new_version' is not sem. version compliant." + fi + + if git tag --list "v*" | grep -qE "^v$new_version$"; then + die "Git tag 'v$new_version' already exists locally." + fi + + # Get all remote versions. + local remote_versions=() + readarray -t remote_versions < \ + <(git ls-remote origin "regs/tags/v*" | cut -f 2 | sed "s@/refs/tags/v@@g") + + # shellcheck disable=SC2128 + if [ "${#remote_versions[@]}" = "0" ]; then + # No version tags yet. Its ok. + return 0 + fi + + if echo "${remote_versions[@]}" | grep "$new_version"; then + die "Remote already contains version tag 'v$new_version'". + fi + + # Sort the versions. + # The top version must be the new one! + latest=$(ci_container_mgr run --rm alpine/semver semver "${remote_versions[@]}" "$new_version" | tail -1) + + if [ "$latest" != "$new_version" ]; then + die "Your version '$new_version' is not newer than the remote ones:" \ + "${remote_versions[@]}" + fi +} + +function main() { + cd "$ROOT_DIR" + + local version="$1" + + local branch + branch=$(git branch --show-current) + + if [ "$branch" != "$RELEASE_BRANCH" ] && [ "${FORCE_RELEASE:-}" != "true" ]; then + die "Can only tag on 'main'. Use 'FORCE_RELEASE=true'." + fi + + if ! git diff --quiet --exit-code; then + die "You have changes on this branch." + fi + + delete_prepare_tags + + local prepare_tag="prepare-v$version" + + check_new_version "$version" + commit_version_file "$version" + + create_prepare_tag "$prepare_tag" + trigger_build "$branch" "$prepare_tag" +} + +main "$@" From 31e4bf8f490dec4e45cb9a53665226b49de30875 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 16:55:52 +0200 Subject: [PATCH 07/18] chore: migrate tooling tripsu->fuzon --- tools/bench/benchmark.sh | 91 +++++++--------------------- tools/build-image.sh | 6 +- tools/build-package.sh | 2 +- tools/ci/create-github-release.sh | 2 +- tools/ci/images/Containerfile | 4 +- tools/ci/upload-ci-images.sh | 2 +- tools/nix/flake.lock | 99 +++++++++++++++++++++++++++++++ tools/nix/flake.nix | 6 +- 8 files changed, 131 insertions(+), 81 deletions(-) create mode 100644 tools/nix/flake.lock diff --git a/tools/bench/benchmark.sh b/tools/bench/benchmark.sh index 30db925..d98eb59 100644 --- a/tools/bench/benchmark.sh +++ b/tools/bench/benchmark.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Benchmark runtime and memory usage of tripsu +# Benchmark runtime and memory usage of fuzon # Compares the working directory version against a baseline branch (main by default) set -euo pipefail @@ -26,52 +26,29 @@ BASE_URL="$(git config --get remote.origin.url)" && cd "${BASE_DIR}" \ && just build "${BUILD_ARGS[@]}" ) -BASE_BIN="${BASE_DIR}/target/${PROFILE}/tripsu" +BASE_BIN="${BASE_DIR}/target/${PROFILE}/fuzon" # current binary COMP_BRANCH="$(git rev-parse --abbrev-ref HEAD)" just build "${BUILD_ARGS[@]}" -COMP_BIN="./target/${PROFILE}/tripsu" +COMP_BIN="./target/${PROFILE}/fuzon" # setup data DATA_URL="https://ftp.uniprot.org/pub/databases/uniprot/current_release/rdf/proteomes.rdf.xz" -INPUT="/tmp/proteomes.nt" +INPUT="/tmp/proteomes.ttl" # Download data if needed if [ ! -f ${INPUT} ]; then curl "${DATA_URL}" \ | xz -dc - \ - | rdfpipe-rs -i rdf-xml -o nt - \ + | rdfpipe-rs -i rdf-xml -o ttl - \ > "${INPUT}" || rm "${INPUT}" fi -# setup config -RULES=$(mktemp) -BASE_IDX=$(mktemp) -COMP_IDX=$(mktemp) - -cat << EOF > "${RULES}" - -nodes: - of_type: - - "http://purl.uniprot.org/core/Proteome" - - "http://purl.uniprot.org/core/Strain" - -objects: - on_type_predicate: - "http://purl.uniprot.org/core/Submission_Citation": - - "http://purl.uniprot.org/core/author" - - on_predicate: - - "http://purl.org/dc/terms/identifier" - -EOF - ### Commands to benchmark -BASE_CMD_IDX="${BASE_BIN} index -o ${BASE_IDX} ${INPUT}" -COMP_CMD_IDX="${COMP_BIN} index -o ${COMP_IDX} ${INPUT}" -BASE_CMD_PSD="${BASE_BIN} pseudo -r ${RULES} -x ${BASE_IDX} ${INPUT}" -COMP_CMD_PSD="${COMP_BIN} pseudo -r ${RULES} -x ${COMP_IDX} ${INPUT}" +QUERY="tein" +BASE_CMD="${BASE_BIN} index -t100 -q ${QUERY} -s ${INPUT}" +COMP_CMD="${COMP_BIN} index -t100 -q ${QUERY} -s ${INPUT}" ### functions for profiling @@ -99,14 +76,12 @@ mem_prof() { } make_report() { - local cpu_index=$1 - local cpu_pseudo=$2 - local mem_index=$3 - local mem_pseudo=$4 - local base_branch=$5 + local cpu=$1 + local mem=$2 + local base_branch=$3 cat <<-MD - # tripsu profiling + # fuzon profiling > date: $(date -u +%Y-%m-%d) @@ -116,25 +91,14 @@ make_report() { Run time compared using hyperfine - ### Indexing - - $(cat "${cpu_index}") - - ### Pseudonymization - - $(cat "${cpu_pseudo}") + $(cat "${cpu}") ## Memory Heap memory usage compared using heaptrack - ### Indexing - - $(cat "${mem_index}") - - ### Pseudonymization + $(cat "${mem}") - $(cat "${mem_pseudo}") MD } @@ -142,30 +106,17 @@ make_report() { ### Run profiling ## Profile cpu time -HYPF_IDX_OUT=$(mktemp) -HYPF_PSD_OUT=$(mktemp) +HYPF_OUT=$(mktemp) -# indexing -cpu_prof "${BASE_BRANCH}" "${BASE_CMD_IDX}" \ - "${COMP_BRANCH}" "${COMP_CMD_IDX}" "${HYPF_IDX_OUT}" -# pseudonymization -cpu_prof "${BASE_BRANCH}" "${BASE_CMD_IDX}" \ - "${COMP_BRANCH}" "${COMP_CMD_IDX}" "${HYPF_PSD_OUT}" +cpu_prof "${BASE_BRANCH}" "${BASE_CMD}" \ + "${COMP_BRANCH}" "${COMP_CMD}" "${HYPF_OUT}" ## Profile memory -HEAP_IDX_OUT=$(mktemp) -HEAP_PSD_OUT=$(mktemp) +HEAP_OUT=$(mktemp) -# indexing -mem_prof "${BASE_BRANCH}" "${BASE_CMD_IDX}" > "${HEAP_IDX_OUT}" -mem_prof "${COMP_BRANCH}" "${COMP_CMD_IDX}" >> "${HEAP_IDX_OUT}" -# pseudonymization -mem_prof "${BASE_BRANCH}" "${BASE_CMD_PSD}" > "${HEAP_PSD_OUT}" -mem_prof "${COMP_BRANCH}" "${COMP_CMD_PSD}" >> "${HEAP_PSD_OUT}" +mem_prof "${BASE_BRANCH}" "${BASE_CMD}" > "${HEAP_OUT}" +mem_prof "${COMP_BRANCH}" "${COMP_CMD}" >> "${HEAP_OUT}" ### Reporting -make_report \ - "${HYPF_IDX_OUT}" "${HYPF_PSD_OUT}" \ - "${HEAP_IDX_OUT}" "${HEAP_PSD_OUT}" \ - "${BASE_BRANCH}" > "${OUTPUT}" +make_report "${HYPF_OUT}" "${HEAP_OUT}" "${BASE_BRANCH}" > "${OUTPUT}" diff --git a/tools/build-image.sh b/tools/build-image.sh index 11459c7..2084faa 100755 --- a/tools/build-image.sh +++ b/tools/build-image.sh @@ -49,8 +49,8 @@ function main() { fi fi - image_name=$(nix eval --raw "./tools/nix#images.tripsu.imageName") - image_tag=$(nix eval --raw "./tools/nix#images.tripsu.imageTag") + image_name=$(nix eval --raw "./tools/nix#images.fuzon.imageName") + image_tag=$(nix eval --raw "./tools/nix#images.fuzon.imageTag") # We cannot use `:` before the tag, because `skopeo` # is unable to read this path correctly, because it @@ -61,7 +61,7 @@ function main() { print_info "Building image '$dir'." nix --version - nix build -L "./tools/nix#images.tripsu" \ + nix build -L "./tools/nix#images.fuzon" \ --out-link "$dir" "${args[@]}" } diff --git a/tools/build-package.sh b/tools/build-package.sh index 3d9d1c1..45be2df 100755 --- a/tools/build-package.sh +++ b/tools/build-package.sh @@ -14,6 +14,6 @@ dir="build/package" print_info "Building the package." nix --version -nix build -L "./tools/nix#tripsu" \ +nix build -L "./tools/nix#fuzon" \ --out-link "$dir" \ "$@" diff --git a/tools/ci/create-github-release.sh b/tools/ci/create-github-release.sh index 2c0102c..f6b9b77 100755 --- a/tools/ci/create-github-release.sh +++ b/tools/ci/create-github-release.sh @@ -18,7 +18,7 @@ function main() { gh release create "$tag" \ --repo="$repo" \ - --title="tripsu: $version" \ + --title="fuzon: $version" \ --generate-notes print_info "Successfully created release. All done." diff --git a/tools/ci/images/Containerfile b/tools/ci/images/Containerfile index 27c23cf..2b1f2eb 100644 --- a/tools/ci/images/Containerfile +++ b/tools/ci/images/Containerfile @@ -4,8 +4,8 @@ # do other non-docker related stuff. FROM alpine:latest as base-podman -LABEL org.opencontainers.image.source https://github.com/sdsc-ordes/tripsu -LABEL org.opencontainers.image.description "CI container image for tripsu" +LABEL org.opencontainers.image.source https://github.com/sdsc-ordes/fuzon +LABEL org.opencontainers.image.description "CI container image for fuzon" LABEL org.opencontainers.image.license "Apache" RUN apk add findutils coreutils git jq curl bash just parallel podman diff --git a/tools/ci/upload-ci-images.sh b/tools/ci/upload-ci-images.sh index d67d092..3afcf72 100755 --- a/tools/ci/upload-ci-images.sh +++ b/tools/ci/upload-ci-images.sh @@ -28,7 +28,7 @@ function build_ci_image() { } tag="${1:-1.0.0}" -repository="${2:-ghcr.io/sdsc-ordes/tripsu}" +repository="${2:-ghcr.io/sdsc-ordes/fuzon}" container_file="$DIR/images/Containerfile" if [ "${CI:-}" = "true" ]; then diff --git a/tools/nix/flake.lock b/tools/nix/flake.lock new file mode 100644 index 0000000..51cc2df --- /dev/null +++ b/tools/nix/flake.lock @@ -0,0 +1,99 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1725432240, + "narHash": "sha256-+yj+xgsfZaErbfYM3T+QvEE2hU7UuE+Jf0fJCJ8uPS0=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "ad416d066ca1222956472ab7d0555a6946746a80", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgsStable": { + "locked": { + "lastModified": 1720535198, + "narHash": "sha256-zwVvxrdIzralnSbcpghA92tWu2DV2lwv89xZc8MTrbg=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "205fd4226592cc83fd4c0885a3e4c9c400efabb5", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "nixpkgsStable": "nixpkgsStable", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1725589472, + "narHash": "sha256-+OB00N6Yql/ZRQQkQ0PNnxfW2tH89DHnv29hBS7tXMM=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "2b00881d2ff72174cffdc007238cb6bedd6e1d8e", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/tools/nix/flake.nix b/tools/nix/flake.nix index 7370a33..24002d7 100644 --- a/tools/nix/flake.nix +++ b/tools/nix/flake.nix @@ -43,7 +43,7 @@ }: let # This is string (without toString it would be a `path` which is put into the store) rootDir = toString ./. + "../../.."; - python = pkgs.python312; + python = nixpkgs.python312; in flake-utils.lib.eachDefaultSystem # Creates an attribute map `{ devShells..default = ...}` @@ -93,7 +93,7 @@ buildInputs = []; # The package of this CLI tool. - # The global version for tripsu. + # The global version for fuzon. # This is gonna get tooled later. fuzon = (import ./pkgs/fuzon.nix) { inherit rootDir rustToolchain pkgs lib; @@ -126,7 +126,7 @@ }; packages = { - tripsu = fuzon; + fuzon = fuzon; images = { ci = (import ./images/ci.nix) { From 07a43e2b2c475b6d51718f7330bb8e266e4c7359 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 17:12:27 +0200 Subject: [PATCH 08/18] chore: target crate path in nix flake --- tools/nix/pkgs/fuzon.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/nix/pkgs/fuzon.nix b/tools/nix/pkgs/fuzon.nix index 29dcfd0..e31dab2 100644 --- a/tools/nix/pkgs/fuzon.nix +++ b/tools/nix/pkgs/fuzon.nix @@ -10,12 +10,12 @@ rustc = rustToolchain; }; - cargoFile = /. + rootDir + "fuzon/Cargo.toml"; - lockFile = /. + rootDir + "fuzon/Cargo.lock"; + cargoFile = /. + rootDir + "/fuzon/Cargo.toml"; + lockFile = /. + rootDir + "/fuzon/Cargo.lock"; in rustPlatform.buildRustPackage { name = "fuzon"; - src = /. + rootDir + "fuzon/"; + src = /. + rootDir; version = (lib.importTOML cargoFile).package.version; From a64856f4c7fa6bb3b2e89fea1a0d36008c4533e1 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 17:18:57 +0200 Subject: [PATCH 09/18] chore: add just recipe for nix dev --- justfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/justfile b/justfile index 606e23e..0e2c53d 100644 --- a/justfile +++ b/justfile @@ -13,9 +13,15 @@ build *args: --release \ {{args}} -# development environment -py-develop *args: +# install editable python package +maturin-dev *args: maturin develop \ --manifest-path pyfuzon/Cargo.toml \ --uv \ {{args}} + +nix-develop *args: + cd {{root}} && \ + cmd=("$@") && \ + { [ -n "${cmd:-}" ] || cmd=("zsh"); } && \ + nix develop ./tools/nix#default --command "${cmd[@]}" From b0136eae1fbc5ca1b7d4dc4524e6a70812fd9e97 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 17:25:04 +0200 Subject: [PATCH 10/18] docs(readme): describe python api --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8616b31..9f3dac8 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,9 @@ To filter the top 3 matches in a file non-interactively: $ fuzon -q 'aspirin' --top 3 -s onto1.ttl -s onto2.ttl ``` +Running fuzon without a query will start an interactive prompt to browse the input ontologies. -Not implemented yet: ~~Running fuzon with a set of RDF ontologies / terminologies will start an interactive prompt using [fzf](https://github.com/junegunn/fzf) to browse the input ontologies.~~ - -### rust crate +### rust library ```rust use fuzon; let r1 = BufReader::new(File::open("onto1.ttl")) @@ -44,3 +43,12 @@ matcher.rank_terms("some query") ### python package +```python +from pyfuzon.matcher import TermMatcher + +matcher = TermMatcher.from_files("https://example.org/onto1.ttl", "/data/onto2.ttl") +matcher.terms #accesses the list of terms loaded from input files +matcher.score("query") # returns the match score of each term for the input query. +matcher.rank("query") # returns the list of terms sorted by similarity with the query. +matcher.top("query", 5) # shows top 5 most similar results (sorted). +``` From 1e3f74950cc9ee6d1ed563e853662f5b4bcd7f98 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 17:27:56 +0200 Subject: [PATCH 11/18] docs(readme): minimal development instructions --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9f3dac8..14f1705 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,7 @@ cargo build --release ## usage -### Command line interface - - +### command line interface To filter the top 3 matches in a file non-interactively: @@ -40,7 +38,6 @@ let matcher = TermMatcher::from_readers(vec![r1, r2]) matcher.rank_terms("some query") ``` - ### python package ```python @@ -52,3 +49,12 @@ matcher.score("query") # returns the match score of each term for the input quer matcher.rank("query") # returns the list of terms sorted by similarity with the query. matcher.top("query", 5) # shows top 5 most similar results (sorted). ``` + +## development + +A nix dev shell with all build dependencies is provided. +Assuming just and nix are installed on the machine, you can enter the shell with: + +```shell +just nix-develop +``` From c38185d441f6132f86089de683401161b1ef7421 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 17:35:32 +0200 Subject: [PATCH 12/18] chore: rm unneeded tooling --- tools/build-image.sh | 68 --------------------- tools/build-package.sh | 19 ------ tools/format-rust.sh | 22 ------- tools/general.sh | 108 --------------------------------- tools/lint-rust.sh | 22 ------- tools/lint-ub-rust.sh | 18 ------ tools/release.sh | 132 ----------------------------------------- 7 files changed, 389 deletions(-) delete mode 100755 tools/build-image.sh delete mode 100755 tools/build-package.sh delete mode 100755 tools/format-rust.sh delete mode 100755 tools/general.sh delete mode 100755 tools/lint-rust.sh delete mode 100755 tools/lint-ub-rust.sh delete mode 100755 tools/release.sh diff --git a/tools/build-image.sh b/tools/build-image.sh deleted file mode 100755 index 2084faa..0000000 --- a/tools/build-image.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090,SC1091 -# -# Build the Nix container image. - -set -euo pipefail - -ROOT_DIR=$(git rev-parse --show-toplevel) -. "$ROOT_DIR/tools/general.sh" - -VERSION_FILE="$ROOT_DIR/Cargo.toml" - -function clean_up() { - if ! ci_is_running; then - # Never change the version file, only - # on explicit `just release`. - print_info "Restoring '$VERSION_FILE'." - git restore "$VERSION_FILE" || true - fi -} - -trap clean_up EXIT - -function main() { - args=("$@") - - if ! ci_is_running || ! ci_is_release; then - print_info "Building image for development." - - # Define the image version from Git SHA - version="0.0.0-dev.$(git rev-parse --short=7 HEAD)" - - # Write the temporary version file (gets restored...) - dasel put -r toml -f "$VERSION_FILE" -t string -v "$version" .package.version - else - print_info "Building image for release." - - # When CI and in Release, the requested version must match. - version=$(dasel -r toml -f "$VERSION_FILE" .package.version -w yaml) - - release_version=${GITHUB_REF##*prepare-v} - - if [ "$version" != "$release_version" ]; then - die "The version '$version' in '$VERSION_FILE' does not corresponds" \ - "with the version '$release_version' to build." \ - "Update the version file to align!" \ - "Nix is pure and cannot rely on Git tags to" \ - "get the version from." - fi - fi - - image_name=$(nix eval --raw "./tools/nix#images.fuzon.imageName") - image_tag=$(nix eval --raw "./tools/nix#images.fuzon.imageTag") - - # We cannot use `:` before the tag, because `skopeo` - # is unable to read this path correctly, because it - # stupidly deals with it. - dir="build/image/$image_name|$image_tag.tar.gz" - - cd "$ROOT_DIR" - - print_info "Building image '$dir'." - nix --version - nix build -L "./tools/nix#images.fuzon" \ - --out-link "$dir" "${args[@]}" -} - -main "$@" diff --git a/tools/build-package.sh b/tools/build-package.sh deleted file mode 100755 index 45be2df..0000000 --- a/tools/build-package.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090,SC1091 -# -# Build the Nix container image. - -set -euo pipefail - -ROOT_DIR=$(git rev-parse --show-toplevel) -. "$ROOT_DIR/tools/general.sh" - -cd "$ROOT_DIR" - -dir="build/package" - -print_info "Building the package." -nix --version -nix build -L "./tools/nix#fuzon" \ - --out-link "$dir" \ - "$@" diff --git a/tools/format-rust.sh b/tools/format-rust.sh deleted file mode 100755 index c2a8d8f..0000000 --- a/tools/format-rust.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090,SC1091 -set -e -set -u - -ROOT_DIR=$(git rev-parse --show-toplevel) -. "$ROOT_DIR/tools/general.sh" - -cd "$ROOT_DIR" - -fmt_args=() -if ci_is_running; then - fmt_args+=("--check") -fi - -cargo --version -cargo fmt --version - -print_info "Run Rust format." -cargo fmt "${fmt_args[@]}" "$@" - -print_info "Done." diff --git a/tools/general.sh b/tools/general.sh deleted file mode 100755 index 068bcd7..0000000 --- a/tools/general.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090,SC1091 -# shellcheck disable=SC2154,SC2086 - -function _print() { - local color="$1" - local flags="$2" - local header="$3" - shift 3 - - local hasColor="0" - if [ "${FORCE_COLOR:-}" != 1 ]; then - [ -t 1 ] && hasColor="1" - else - hasColor="1" - fi - - if [ "$hasColor" = "0" ] || [ "${LOG_COLORS:-}" = "false" ]; then - local msg - msg=$(printf '%b\n' "$@") - msg="${msg//$'\n'/$'\n' }" - echo $flags -e "-- $header$msg" - else - local s=$'\033' e='[0m' - local msg - msg=$(printf "%b\n" "$@") - msg="${msg//$'\n'/$'\n' }" - echo $flags -e "${s}${color}-- $header$msg${s}${e}" - fi -} -function print_info() { - _print "[0;94m" "" "" "$@" -} - -function print_warning() { - _print "[0;31m" "" "WARN: " "$@" >&2 -} - -function print_error() { - _print "[0;31m" "" "ERROR: " "$@" >&2 -} - -function die() { - print_error "$@" - exit 1 -} - -function ci_is_running() { - if [ "${CI:-}" = "true" ]; then - return 0 - fi - - return 1 -} - -function ci_is_release() { - if [ "${CI_IS_RELEASE:-}" = "true" ]; then - return 0 - fi - - return 1 -} - -function ci_setup_git() { - git config --global user.name "SDSC CI" - git config --global user.email "ci@sdsc.ethz.ch" -} - -function ci_setup_nix() { - local install_prefix="${1:-/usr/sbin}" - - print_info "Install Nix." - apk add curl bash xz shadow - sh <(curl -L https://nixos.org/nix/install) --daemon --yes - cp /root/.nix-profile/bin/* "$install_prefix/" - - print_info "Enable Features for Nix." - mkdir -p ~/.config/nix - { - echo "experimental-features = nix-command flakes" - echo "accept-flake-config = true" - } >~/.config/nix/nix.conf -} - -# Run the container manager which is defined. -# in env. variable `CONTAINER_MGR` -# (by default `podman` if existing). -function ci_container_mgr() { - local mgr="${CONTAINER_MGR:-podman}" - - if command -v "$mgr" &>/dev/null; then - echo -e "Running '$mgr' as:\n$(printf "'%s' " "podman" "$@")" >&2 - "$mgr" "$@" - else - echo -e "Running docker as:\n$(printf "'%s' " "docker" "$@")" - docker "$@" - fi -} - -function ci_container_mgr_login() { - local user="$1" - local token="$2" - - [ -n "$token" ] || die "Docker login token is empty" - echo "$token" | - ci_container_mgr login --password-stdin --username "$user" || - die "Could not log into docker." -} diff --git a/tools/lint-rust.sh b/tools/lint-rust.sh deleted file mode 100755 index 848c4d8..0000000 --- a/tools/lint-rust.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090,SC1091 -set -e -set -u - -ROOT_DIR=$(git rev-parse --show-toplevel) -. "$ROOT_DIR/tools/general.sh" - -cd "$ROOT_DIR" - -cargo --version -cargo clippy --version - -print_info "Run Rust Clippy linter." - -cargo clippy --no-deps -- -D warnings -A clippy::needless_return "$@" || - { - git diff --name-status || true - die "Rust clippy failed." - } - -print_info "Done." diff --git a/tools/lint-ub-rust.sh b/tools/lint-ub-rust.sh deleted file mode 100755 index 71c1725..0000000 --- a/tools/lint-ub-rust.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090,SC1091 -set -e -set -u - -ROOT_DIR=$(git rev-parse --show-toplevel) -. "$ROOT_DIR/tools/general.sh" - -cd "$ROOT_DIR" - -cargo --version -cargo miri --version - -print_info "Run Rust Miri to check undefined behaviour." -cargo miri test "$@" || - die "Rust Miri failed." - -print_info "Done." diff --git a/tools/release.sh b/tools/release.sh deleted file mode 100755 index ba3ef13..0000000 --- a/tools/release.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090,SC1091 -# -# Creating a prepare tag to trigger the release process on the -# Github workflow. Can only be called on `main`. -# -# Usage: release.sh "1.2.0" - -set -euo pipefail - -ROOT_DIR=$(git rev-parse --show-toplevel) -. "$ROOT_DIR/tools/general.sh" - -RELEASE_BRANCH="${RELEASE_BRANCH:-main}" -VERSION_FILE="$ROOT_DIR/Cargo.toml" - -function delete_prepare_tags() { - readarray -t prepareTag < <(git tag --list "prepare-*") - - for tag in "${prepareTag[@]}"; do - print_info "Deleting prepare tag '$tag'." - git push -f origin ":${tag}" || true - git tag -d "$tag" || die - done -} - -function create_prepare_tag() { - local tag="$1" - - print_info "Tagging with '$tag'." - git tag -a -m "Version $version" "$tag" || die "Could not create tag." - - print_info "Tag contains:" - git cat-file -p "$tag" || die "Could not show tag content." - - print_info "Successfully created prepate tag '$tag'." -} - -function commit_version_file() { - local version="$1" - print_info "Writing new version file... (for Nix)" - - dasel put -r toml -f "$VERSION_FILE" -t string -v "$version" .package.version - - if ! git diff --exit-code --quiet; then - # Commit if we have change. - git add "$VERSION_FILE" - git commit -m "chore: release '$version'" - fi -} - -function trigger_build() { - local branch="$1" - local tag="$2" - - printf "Do you want to trigger the build? [y|n]: " - read -r answer - if [ "$answer" != "y" ]; then - die "Do not trigger build -> abort." - fi - - print_info "Pushing tag '$tag'." - git push -f origin --no-follow-tags "$branch" "$tag" - - print_info "Successfully triggered build." -} - -function check_new_version() { - local new_version="$1" - - # Check that is a version. - if [ "$(ci_container_mgr run --rm docker.io/alpine/semver semver "$new_version" | tail -1)" != "$new_version" ]; then - die "Your version '$new_version' is not sem. version compliant." - fi - - if git tag --list "v*" | grep -qE "^v$new_version$"; then - die "Git tag 'v$new_version' already exists locally." - fi - - # Get all remote versions. - local remote_versions=() - readarray -t remote_versions < \ - <(git ls-remote origin "regs/tags/v*" | cut -f 2 | sed "s@/refs/tags/v@@g") - - # shellcheck disable=SC2128 - if [ "${#remote_versions[@]}" = "0" ]; then - # No version tags yet. Its ok. - return 0 - fi - - if echo "${remote_versions[@]}" | grep "$new_version"; then - die "Remote already contains version tag 'v$new_version'". - fi - - # Sort the versions. - # The top version must be the new one! - latest=$(ci_container_mgr run --rm alpine/semver semver "${remote_versions[@]}" "$new_version" | tail -1) - - if [ "$latest" != "$new_version" ]; then - die "Your version '$new_version' is not newer than the remote ones:" \ - "${remote_versions[@]}" - fi -} - -function main() { - cd "$ROOT_DIR" - - local version="$1" - - local branch - branch=$(git branch --show-current) - - if [ "$branch" != "$RELEASE_BRANCH" ] && [ "${FORCE_RELEASE:-}" != "true" ]; then - die "Can only tag on 'main'. Use 'FORCE_RELEASE=true'." - fi - - if ! git diff --quiet --exit-code; then - die "You have changes on this branch." - fi - - delete_prepare_tags - - local prepare_tag="prepare-v$version" - - check_new_version "$version" - commit_version_file "$version" - - create_prepare_tag "$prepare_tag" - trigger_build "$branch" "$prepare_tag" -} - -main "$@" From e05c8aadeba25dc876ae30a7a6e79d1a1890de15 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 18:31:30 +0200 Subject: [PATCH 13/18] refactor(nix): simplify dev image setup --- justfile | 29 ++++++++++++++++++++++++----- tools/nix/flake.nix | 13 +------------ tools/nix/images/ci.nix | 22 ---------------------- tools/nix/images/dev.nix | 10 ++++++++++ 4 files changed, 35 insertions(+), 39 deletions(-) delete mode 100644 tools/nix/images/ci.nix create mode 100644 tools/nix/images/dev.nix diff --git a/justfile b/justfile index 0e2c53d..b54bda6 100644 --- a/justfile +++ b/justfile @@ -20,8 +20,27 @@ maturin-dev *args: --uv \ {{args}} -nix-develop *args: - cd {{root}} && \ - cmd=("$@") && \ - { [ -n "${cmd:-}" ] || cmd=("zsh"); } && \ - nix develop ./tools/nix#default --command "${cmd[@]}" +develop-nix *args: + cd {{root}} \ + && cmd=("$@") \ + && { [ -n "${cmd:-}" ] || cmd=("zsh"); } \ + && nix develop ./tools/nix#default --command "${cmd[@]}" + +develop-docker: + docker run \ + --user 1000:1000 \ + -it \ + -w "/build/workspace" \ + --mount type=bind,source="$(pwd)",target=/build/work \ + {{image}}:dev + +# maintenance + +image-build: + nix build -L "./tools/nix#image.dev" --out-link "target/image.dev" \ + && docker load < "build/image.dev" + nix build -L "./tools/nix#image.fuzon" --out-link "target/image.fuzon" \ + && docker load < "build/image.fuzon" + + + diff --git a/tools/nix/flake.nix b/tools/nix/flake.nix index 24002d7..64bafc7 100644 --- a/tools/nix/flake.nix +++ b/tools/nix/flake.nix @@ -112,24 +112,13 @@ ++ benchInputs; }; - ci = mkShell { - inherit buildInputs; - nativeBuildInputs = nativeBuildInputsBasic ++ nativeBuildInputsDev; - - # Due to some weird handling of TMPDIR inside containers: - # https://github.com/NixOS/nix/issues/8355 - # We have to reset the TMPDIR to make `nix build` work inside - # a development shell. - # Without `nix develop` it works. - shellHook = "unset TMPDIR"; - }; }; packages = { fuzon = fuzon; images = { - ci = (import ./images/ci.nix) { + dev = (import ./images/dev.nix) { inherit pkgs; devShellDrv = devShells.default; }; diff --git a/tools/nix/images/ci.nix b/tools/nix/images/ci.nix deleted file mode 100644 index ced142d..0000000 --- a/tools/nix/images/ci.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ - pkgs, - devShellDrv, - ... -}: let - version = "1.0.0"; # The version of these CI images. - image_name = "ghcr.io/sdsc-ordes/fuzon"; - - buildImage = type: - pkgs.dockerTools.buildNixShellImage { - name = image_name; - tag = "ci-${type}-${version}"; - drv = devShellDrv; - }; -in rec { - format = buildImage "format"; - lint = buildImage "lint"; - build = buildImage "build"; - test = buildImage "test"; - package = buildImage "package"; - deploy = buildImage "deploy"; -} diff --git a/tools/nix/images/dev.nix b/tools/nix/images/dev.nix new file mode 100644 index 0000000..5734d92 --- /dev/null +++ b/tools/nix/images/dev.nix @@ -0,0 +1,10 @@ +{ pkgs, devShellDrv, ... }: + +let + image_name = "ghcr.io/sdsc-ordes/fuzon"; + +in pkgs.dockerTools.buildNixShellImage { + name = image_name; + tag = "dev"; + drv = devShellDrv; +} From af6bd5b12c532238f403f8c1f4758e350e61c871 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 6 Sep 2024 18:34:38 +0200 Subject: [PATCH 14/18] chore: add maintenance just recipe --- justfile | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/justfile b/justfile index b54bda6..10878d9 100644 --- a/justfile +++ b/justfile @@ -1,7 +1,9 @@ set positional-arguments set shell := ["bash", "-cue"] +image := "ghcr.io/sdsc-ordes/fuzon" root := justfile_directory() +## build # build all packages build *args: @@ -20,12 +22,16 @@ maturin-dev *args: --uv \ {{args}} +## development + +# enter nix devshell develop-nix *args: cd {{root}} \ && cmd=("$@") \ && { [ -n "${cmd:-}" ] || cmd=("zsh"); } \ && nix develop ./tools/nix#default --command "${cmd[@]}" +# enter development container develop-docker: docker run \ --user 1000:1000 \ @@ -34,13 +40,18 @@ develop-docker: --mount type=bind,source="$(pwd)",target=/build/work \ {{image}}:dev -# maintenance +## maintenance -image-build: +# build images +docker-build: nix build -L "./tools/nix#image.dev" --out-link "target/image.dev" \ && docker load < "build/image.dev" nix build -L "./tools/nix#image.fuzon" --out-link "target/image.fuzon" \ && docker load < "build/image.fuzon" +# push images +docker-push: docker-build + docker push {{image}}:dev + docker push {{image}}:fuzon From e7934098857b5d8b70231058bb42916c071ef845 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Tue, 10 Sep 2024 15:32:49 +0200 Subject: [PATCH 15/18] fix(just): image file path --- justfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/justfile b/justfile index 10878d9..4c16291 100644 --- a/justfile +++ b/justfile @@ -44,10 +44,10 @@ develop-docker: # build images docker-build: - nix build -L "./tools/nix#image.dev" --out-link "target/image.dev" \ - && docker load < "build/image.dev" - nix build -L "./tools/nix#image.fuzon" --out-link "target/image.fuzon" \ - && docker load < "build/image.fuzon" + nix build -L "./tools/nix#images.dev" --out-link "target/image.dev" \ + && docker load < "target/image.dev" + nix build -L "./tools/nix#images.fuzon" --out-link "target/image.fuzon" \ + && docker load < "target/image.fuzon" # push images docker-push: docker-build From b253aca7d5ce302bbe6179c6481a33501d38cf8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Thu, 12 Sep 2024 17:19:24 +0200 Subject: [PATCH 16/18] chore: Nix fixes (#12) * chore: nix builds now but wont find python * fix: pass rootSrc to derivation `fuzon` * fix: correct `python312` dependencies --- .gitignore | 14 +- Cargo.lock | 1687 ++++++++++++++++++++++++++++++ justfile | 24 +- tools/nix/flake.nix | 80 +- tools/nix/pkgs/fuzon.nix | 32 - tools/nix/pkgs/fuzon/default.nix | 41 + 6 files changed, 1781 insertions(+), 97 deletions(-) create mode 100644 Cargo.lock delete mode 100644 tools/nix/pkgs/fuzon.nix create mode 100644 tools/nix/pkgs/fuzon/default.nix diff --git a/.gitignore b/.gitignore index 70a6345..2faa1fd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +.direnv +package + # Generated by Cargo # will have compiled files and executables debug/ @@ -5,23 +8,12 @@ target/ __pycache__/ *.so -# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries -# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -Cargo.lock - # These are backup files generated by rustfmt **/*.rs.bk -# MSVC Windows builds of rustc generate these, which store debugging information -*.pdb - # RustRover # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ - -# Added by cargo - -/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..ccaa4e1 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1687 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anstream" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" + +[[package]] +name = "anstyle-parse" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e1496f8fb1fbf272686b8d37f523dab3e4a7443300055e74cdaa449f3114356" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim 0.8.0", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "clap" +version = "4.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_derive" +version = "4.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + +[[package]] +name = "colorchoice" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" + +[[package]] +name = "compact_str" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6050c3a16ddab2e412160b31f2c871015704239bca62f72f6e5f0be631d3f644" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.6.0", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "fuzon" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.17", + "crossterm", + "lazy_static", + "oxrdf 0.1.7", + "oxttl", + "ratatui", + "reqwest", + "rff", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" + +[[package]] +name = "hyper" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + +[[package]] +name = "instability" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b23a0c8dfe501baac4adf6ebbfa6eddf8f0c07f56b058cc1288017e32397846c" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "ipnet" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "lru" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "log", + "wasi", + "windows-sys", +] + +[[package]] +name = "object" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "oxilangtag" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f3f87617a86af77fa3691e6350483e7154c2ead9f1261b75130e21ca0f8acb" +dependencies = [ + "serde", +] + +[[package]] +name = "oxiri" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85d9439ace287894b327bd5522d4f3d813311c719143a1af37826c6a12f808d0" + +[[package]] +name = "oxrdf" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "309287c8a757e25e06a6156acbd73770bac3e319123f5b4bc0a42e232caf97a5" +dependencies = [ + "oxilangtag", + "oxiri", + "rand", +] + +[[package]] +name = "oxrdf" +version = "0.2.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2c3036a00891f976f347c8ac2c3aaec997ceb9367cb79afea99a6bb51d7403d" +dependencies = [ + "oxilangtag", + "oxiri", + "rand", + "thiserror", +] + +[[package]] +name = "oxttl" +version = "0.1.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524df0e2849790b81d132ffd75c233f47a818e6131c0a223e313fccb2a1ac01f" +dependencies = [ + "memchr", + "oxilangtag", + "oxiri", + "oxrdf 0.2.0-rc.1", + "thiserror", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "portable-atomic" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyfuzon" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.17", + "crossterm", + "fuzon", + "lazy_static", + "oxrdf 0.1.7", + "oxttl", + "pyo3", + "ratatui", + "reqwest", + "rff", +] + +[[package]] +name = "pyo3" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831e8e819a138c36e212f3af3fd9eeffed6bf1510a805af35b0edee5ffa59433" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e8730e591b14492a8945cdff32f089250b05f5accecf74aeddf9e8272ce1fa8" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e97e919d2df92eb88ca80a037969f44e5e70356559654962cbb3316d00300c6" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb57983022ad41f9e683a599f2fd13c3664d7063a3ac5714cae4b7bee7d3f206" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec480c0c51ddec81019531705acac51bcdbeae563557c982aa8263bb96880372" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "ratatui" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdef7f9be5c0122f890d58bdf4d964349ba6a6161f705907526d891efabba57d" +dependencies = [ + "bitflags 2.6.0", + "cassowary", + "compact_str", + "crossterm", + "instability", + "itertools", + "lru", + "paste", + "strum", + "strum_macros", + "unicode-segmentation", + "unicode-truncate", + "unicode-width", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" +dependencies = [ + "bitflags 2.6.0", +] + +[[package]] +name = "reqwest" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-registry", +] + +[[package]] +name = "rff" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce6626007d6253f887f6767a61efc034b45498ecf8d8371c7c25388e8a99d46" +dependencies = [ + "clap 2.34.0", + "libc", + "rayon", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustix" +version = "0.38.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "signal-hook" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "syn" +version = "2.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +dependencies = [ + "backtrace", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "unicode-width" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" + +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "url" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + +[[package]] +name = "web-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/justfile b/justfile index 4c16291..0cf8434 100644 --- a/justfile +++ b/justfile @@ -3,9 +3,9 @@ set shell := ["bash", "-cue"] image := "ghcr.io/sdsc-ordes/fuzon" root := justfile_directory() -## build +## Build -# build all packages +# Build all packages. build *args: cargo build \ --manifest-path fuzon/Cargo.toml \ @@ -15,23 +15,27 @@ build *args: --release \ {{args}} -# install editable python package +# Install editable python package. maturin-dev *args: maturin develop \ --manifest-path pyfuzon/Cargo.toml \ --uv \ {{args}} -## development +package-nix *args: + cd {{root}} && \ + nix build "./tools/nix#fuzon" -o "package/fuzon" {{args}} -# enter nix devshell +## Development + +# Enter nix devshell. develop-nix *args: cd {{root}} \ && cmd=("$@") \ && { [ -n "${cmd:-}" ] || cmd=("zsh"); } \ && nix develop ./tools/nix#default --command "${cmd[@]}" -# enter development container +# Enter development container develop-docker: docker run \ --user 1000:1000 \ @@ -40,18 +44,16 @@ develop-docker: --mount type=bind,source="$(pwd)",target=/build/work \ {{image}}:dev -## maintenance +## Maintenance -# build images +# Build images. docker-build: nix build -L "./tools/nix#images.dev" --out-link "target/image.dev" \ && docker load < "target/image.dev" nix build -L "./tools/nix#images.fuzon" --out-link "target/image.fuzon" \ && docker load < "target/image.fuzon" -# push images +# Push images. docker-push: docker-build docker push {{image}}:dev docker push {{image}}:fuzon - - diff --git a/tools/nix/flake.nix b/tools/nix/flake.nix index 64bafc7..a3f5eb6 100644 --- a/tools/nix/flake.nix +++ b/tools/nix/flake.nix @@ -36,25 +36,23 @@ }; outputs = { + self, nixpkgs, flake-utils, rust-overlay, ... - }: let - # This is string (without toString it would be a `path` which is put into the store) - rootDir = toString ./. + "../../.."; - python = nixpkgs.python312; - in + }: flake-utils.lib.eachDefaultSystem # Creates an attribute map `{ devShells..default = ...}` # by calling this function: ( system: let + rootSrc = ./../..; + overlays = [(import rust-overlay)]; # Import nixpkgs and load it into pkgs. # Overlay the rust toolchain - lib = nixpkgs.lib; pkgs = import nixpkgs { inherit system overlays; }; @@ -63,7 +61,7 @@ rustToolchain = pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ../../rust-toolchain.toml; # Things needed only at compile-time. - nativeBuildInputsBasic = with pkgs; [ + basic-deps = with pkgs; [ maturin findutils coreutils @@ -75,59 +73,55 @@ ]; # Things needed only at compile-time. - nativeBuildInputsDev = with pkgs; [ + general-deps = [ rustToolchain - cargo-watch - just + pkgs.cargo-watch + pkgs.just - skopeo - dasel + pkgs.skopeo + pkgs.dasel + pkgs.python313 ]; - benchInputs = with pkgs; [ + benchmark-deps = with pkgs; [ hyperfine heaptrack ]; - # Things needed at runtime. - buildInputs = []; - # The package of this CLI tool. # The global version for fuzon. # This is gonna get tooled later. - fuzon = (import ./pkgs/fuzon.nix) { - inherit rootDir rustToolchain pkgs lib; + fuzon = pkgs.callPackage ./pkgs/fuzon { + inherit rootSrc; + inherit rustToolchain; }; - in - with pkgs; rec { - devShells = { - default = mkShell { - inherit buildInputs; - nativeBuildInputs = nativeBuildInputsBasic ++ nativeBuildInputsDev; - }; - bench = mkShell { - inherit buildInputs; - nativeBuildInputs = nativeBuildInputsBasic - ++ nativeBuildInputsDev - ++ benchInputs; - }; - + in rec { + devShells = { + default = pkgs.mkShell { + packages = basic-deps ++ general-deps; }; + bench = pkgs.mkShell { + packages = + basic-deps + ++ general-deps + ++ benchmark-deps; + }; + }; - packages = { - fuzon = fuzon; + packages = { + fuzon = fuzon; - images = { - dev = (import ./images/dev.nix) { - inherit pkgs; - devShellDrv = devShells.default; - }; + images = { + dev = (import ./images/dev.nix) { + inherit pkgs; + devShellDrv = devShells.default; + }; - fuzon = (import ./images/fuzon.nix) { - inherit pkgs fuzon; - }; + fuzon = (import ./images/fuzon.nix) { + inherit pkgs fuzon; }; }; - } + }; + } ); } diff --git a/tools/nix/pkgs/fuzon.nix b/tools/nix/pkgs/fuzon.nix deleted file mode 100644 index e31dab2..0000000 --- a/tools/nix/pkgs/fuzon.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ - pkgs, - lib, - rustToolchain, - rootDir, - ... -}: let - rustPlatform = pkgs.makeRustPlatform { - cargo = rustToolchain; - rustc = rustToolchain; - }; - - cargoFile = /. + rootDir + "/fuzon/Cargo.toml"; - lockFile = /. + rootDir + "/fuzon/Cargo.lock"; -in - rustPlatform.buildRustPackage { - name = "fuzon"; - src = /. + rootDir; - - version = (lib.importTOML cargoFile).package.version; - - cargoLock = { - inherit lockFile; - }; - - meta = { - description = "A CLI tool to fuzzy search ontology terms by their labels."; - homepage = "https://github.com/sdsc-ordes/fuzon"; - license = lib.licenses.asl20; - maintainers = ["gabyx" "cmdoret"]; - }; - } diff --git a/tools/nix/pkgs/fuzon/default.nix b/tools/nix/pkgs/fuzon/default.nix new file mode 100644 index 0000000..23fd07b --- /dev/null +++ b/tools/nix/pkgs/fuzon/default.nix @@ -0,0 +1,41 @@ +{ + rootSrc, + lib, + makeRustPlatform, + rustToolchain, + python313, +}: let + rustPlatform = makeRustPlatform { + cargo = rustToolchain; + rustc = rustToolchain; + }; + + fs = lib.fileset; + cargoFile = "${rootSrc}/fuzon/Cargo.toml"; + lockFile = "${rootSrc}/Cargo.lock"; +in + (rustPlatform.buildRustPackage + { + name = "fuzon"; + + src = fs.toSource { + root = rootSrc; + fileset = fs.gitTracked rootSrc; + }; + + version = (lib.importTOML cargoFile).package.version; + + cargoLock = { + inherit lockFile; + }; + + meta = { + description = "A CLI tool to fuzzy search ontology terms by their labels."; + homepage = "https://github.com/sdsc-ordes/fuzon"; + license = lib.licenses.asl20; + maintainers = ["gabyx" "cmdoret"]; + }; + }) + .overrideAttrs (finalAttrs: prevAttrs: { + nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [python313]; + }) From 34f44508725cabf5d01a13fdb2b7adfd6f7955ab Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 13 Sep 2024 15:25:26 +0200 Subject: [PATCH 17/18] docs(readme): update dev instructions --- README.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 14f1705..73a13a8 100644 --- a/README.md +++ b/README.md @@ -56,5 +56,26 @@ A nix dev shell with all build dependencies is provided. Assuming just and nix are installed on the machine, you can enter the shell with: ```shell -just nix-develop +just develop-nix ``` + +Alternatively, docker can be used as a development shell: + +```shell +just develop-docker +``` + +Once inside a development shell, the python+rust packages can be built with: + +```shell +just build +``` + +Or the python bindings can be installed in editable mode using: + +```shell +just maturin-dev +# pyfuzon now available in python shells +``` + + From fdeafff392a448e7e8781fbf6820abc4726c8272 Mon Sep 17 00:00:00 2001 From: cmdoret Date: Fri, 13 Sep 2024 15:25:38 +0200 Subject: [PATCH 18/18] chore(just): fix docker mount path --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 0cf8434..d317502 100644 --- a/justfile +++ b/justfile @@ -41,7 +41,7 @@ develop-docker: --user 1000:1000 \ -it \ -w "/build/workspace" \ - --mount type=bind,source="$(pwd)",target=/build/work \ + --mount type=bind,source="$(pwd)",target=/build/workspace \ {{image}}:dev ## Maintenance