From 92a3b4b03d30af57fbd88dc95924cf13c9179c78 Mon Sep 17 00:00:00 2001 From: Max West Date: Thu, 4 Apr 2024 10:56:06 -0700 Subject: [PATCH 1/2] copier update --- .copier-answers.yml | 12 +-- .git_archival.txt | 4 + .gitattributes | 24 +++++ .github/ISSUE_TEMPLATE/0-general_issue.md | 8 ++ .github/ISSUE_TEMPLATE/1-bug_report.md | 17 ++++ .github/ISSUE_TEMPLATE/2-feature_request.md | 18 ++++ .github/dependabot.yml | 10 ++ .github/workflows/asv-main.yml | 101 ++++++++++++++++++++ .github/workflows/asv-nightly.yml | 93 ++++++++++++++++++ .github/workflows/asv-pr.yml | 86 +++++++++++++++++ .github/workflows/build-documentation.yml | 8 +- .github/workflows/linting.yml | 36 ------- .github/workflows/pre-commit-ci.yml | 35 +++++++ .github/workflows/publish-benchmarks-pr.yml | 53 ++++++++++ .github/workflows/publish-to-pypi.yml | 14 ++- .github/workflows/smoke-test.yml | 16 ++-- .github/workflows/testing-and-coverage.yml | 15 +-- .gitignore | 13 ++- .pre-commit-config.yaml | 80 +++++++--------- .setup_dev.sh | 42 ++++++++ benchmarks/__init__.py | 0 benchmarks/asv.conf.json | 80 ++++++++++++++++ benchmarks/benchmarks.py | 16 ++++ docs/conf.py | 27 ++++-- docs/index.rst | 37 +++++++ docs/notebooks/intro_notebook.ipynb | 79 +++++---------- docs/requirements.txt | 15 +-- pyproject.toml | 70 +++++++++++--- src/koffi/example_benchmarks.py | 14 +++ 29 files changed, 828 insertions(+), 195 deletions(-) create mode 100644 .git_archival.txt create mode 100644 .gitattributes create mode 100644 .github/ISSUE_TEMPLATE/0-general_issue.md create mode 100644 .github/ISSUE_TEMPLATE/1-bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/2-feature_request.md create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/asv-main.yml create mode 100644 .github/workflows/asv-nightly.yml create mode 100644 .github/workflows/asv-pr.yml delete mode 100644 .github/workflows/linting.yml create mode 100644 .github/workflows/pre-commit-ci.yml create mode 100644 .github/workflows/publish-benchmarks-pr.yml create mode 100644 .setup_dev.sh create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/asv.conf.json create mode 100644 benchmarks/benchmarks.py create mode 100644 src/koffi/example_benchmarks.py diff --git a/.copier-answers.yml b/.copier-answers.yml index acaf4d5..83c2767 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,16 +1,10 @@ # Changes here will be overwritten by Copier -_commit: v1.4.1 +_commit: v2.0.1 _src_path: gh:lincc-frameworks/python-project-template author_email: maxwest@uw.edu author_name: Maxine West -create_example_module: false -custom_install: true -include_docs: true -include_notebooks: true -mypy_type_checking: none +custom_install: false package_name: koffi -preferred_linter: black project_license: BSD project_name: koffi -use_gitlfs: none -use_isort: true +project_organization: lincc-frameworks diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 0000000..b1a286b --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1,4 @@ +node: $Format:%H$ +node-date: $Format:%cI$ +describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ +ref-names: $Format:%D$ \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..343a755 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,24 @@ +# For explanation of this file and uses see +# https://git-scm.com/docs/gitattributes +# https://developer.lsst.io/git/git-lfs.html#using-git-lfs-enabled-repositories +# https://lincc-ppt.readthedocs.io/en/latest/practices/git-lfs.html +# +# Used by https://github.com/lsst/afwdata.git +# *.boost filter=lfs diff=lfs merge=lfs -text +# *.dat filter=lfs diff=lfs merge=lfs -text +# *.fits filter=lfs diff=lfs merge=lfs -text +# *.gz filter=lfs diff=lfs merge=lfs -text +# +# apache parquet files +# *.parq filter=lfs diff=lfs merge=lfs -text +# +# sqlite files +# *.sqlite3 filter=lfs diff=lfs merge=lfs -text +# +# gzip files +# *.gz filter=lfs diff=lfs merge=lfs -text +# +# png image files +# *.png filter=lfs diff=lfs merge=lfs -text + +.git_archival.txt export-subst \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/0-general_issue.md b/.github/ISSUE_TEMPLATE/0-general_issue.md new file mode 100644 index 0000000..84bb0d7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/0-general_issue.md @@ -0,0 +1,8 @@ +--- +name: General issue +about: Quickly create a general issue +title: '' +labels: '' +assignees: '' + +--- \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/1-bug_report.md b/.github/ISSUE_TEMPLATE/1-bug_report.md new file mode 100644 index 0000000..220a63d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/1-bug_report.md @@ -0,0 +1,17 @@ +--- +name: Bug report +about: Tell us about a problem to fix +title: 'Short description' +labels: 'bug' +assignees: '' + +--- +**Bug report** + + +**Before submitting** +Please check the following: + +- [ ] I have described the situation in which the bug arose, including what code was executed, information about my environment, and any applicable data others will need to reproduce the problem. +- [ ] I have included available evidence of the unexpected behavior (including error messages, screenshots, and/or plots) as well as a descriprion of what I expected instead. +- [ ] If I have a solution in mind, I have provided an explanation and/or pseudocode and/or task list. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/2-feature_request.md b/.github/ISSUE_TEMPLATE/2-feature_request.md new file mode 100644 index 0000000..908ff72 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2-feature_request.md @@ -0,0 +1,18 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: 'Short description' +labels: 'enhancement' +assignees: '' + +--- + +**Feature request** + + +**Before submitting** +Please check the following: + +- [ ] I have described the purpose of the suggested change, specifying what I need the enhancement to accomplish, i.e. what problem it solves. +- [ ] I have included any relevant links, screenshots, environment information, and data relevant to implementing the requested feature, as well as pseudocode for how I want to access the new functionality. +- [ ] If I have ideas for how the new feature could be implemented, I have provided explanations and/or pseudocode and/or task lists for the steps. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..3b5ca19 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" diff --git a/.github/workflows/asv-main.yml b/.github/workflows/asv-main.yml new file mode 100644 index 0000000..f6a6f29 --- /dev/null +++ b/.github/workflows/asv-main.yml @@ -0,0 +1,101 @@ +# This workflow will run benchmarks with airspeed velocity (asv), +# store the new results in the "benchmarks" branch and publish them +# to a dashboard on GH Pages. + +name: Run ASV benchmarks for main + +on: + push: + branches: [ main ] + +env: + PYTHON_VERSION: "3.10" + WORKING_DIR: ${{ github.workspace }}/benchmarks + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + + setup-python: + runs-on: ubuntu-latest + + steps: + - name: Cache Python ${{ env.PYTHON_VERSION }} + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-${{ env.PYTHON_VERSION }} + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + asv-main: + runs-on: ubuntu-latest + needs: setup-python + + permissions: + contents: write + + defaults: + run: + working-directory: ${{ env.WORKING_DIR }} + + steps: + - name: Checkout main branch of the repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Cache Python ${{ env.PYTHON_VERSION }} + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-${{ env.PYTHON_VERSION }} + + - name: Install dependencies + run: | + sudo apt-get update + python -m pip install --upgrade pip + pip install asv==0.6.1 virtualenv tabulate + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Create ASV machine config file + run: asv machine --machine gh-runner --yes + + - name: Fetch previous results from the "benchmarks" branch + run: | + if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then + git merge origin/benchmarks \ + --allow-unrelated-histories \ + --no-commit + mv ../_results . + fi + + - name: Run ASV for the main branch + run: asv run ALL --skip-existing --verbose || true + + - name: Submit new results to the "benchmarks" branch + uses: JamesIves/github-pages-deploy-action@v4 + with: + branch: benchmarks + folder: ${{ env.WORKING_DIR }}/_results + target-folder: _results + + - name: Generate dashboard HTML + run: | + asv show + asv publish + + - name: Deploy to Github pages + uses: JamesIves/github-pages-deploy-action@v4 + with: + branch: gh-pages + folder: ${{ env.WORKING_DIR }}/_html \ No newline at end of file diff --git a/.github/workflows/asv-nightly.yml b/.github/workflows/asv-nightly.yml new file mode 100644 index 0000000..80a2d78 --- /dev/null +++ b/.github/workflows/asv-nightly.yml @@ -0,0 +1,93 @@ +# This workflow will run daily at 06:45. +# It will run benchmarks with airspeed velocity (asv) +# and compare performance with the previous nightly build. + +name: Run benchmarks nightly job + +on: + schedule: + - cron: 45 6 * * * + workflow_dispatch: + +env: + PYTHON_VERSION: "3.10" + WORKING_DIR: ${{ github.workspace }}/benchmarks + NIGHTLY_HASH_FILE: nightly-hash + +jobs: + + asv-nightly: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: ${{ env.WORKING_DIR }} + + steps: + - name: Checkout main branch of the repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Cache Python ${{ env.PYTHON_VERSION }} + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-${{ env.PYTHON_VERSION }} + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Install dependencies + run: | + sudo apt-get update + python -m pip install --upgrade pip + pip install asv==0.6.1 virtualenv + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Create ASV machine config file + run: asv machine --machine gh-runner --yes + + - name: Fetch previous results from the "benchmarks" branch + run: | + if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then + git merge origin/benchmarks \ + --allow-unrelated-histories \ + --no-commit + mv ../_results . + fi + + - name: Get nightly dates under comparison + id: nightly-dates + run: | + echo "yesterday=$(date -d yesterday +'%Y-%m-%d')" >> $GITHUB_OUTPUT + echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Use last nightly commit hash from cache + uses: actions/cache@v4 + with: + path: ${{ env.WORKING_DIR }} + key: nightly-results-${{ steps.nightly-dates.outputs.yesterday }} + + - name: Run comparison of main against last nightly build + run: | + HASH_FILE=${{ env.NIGHTLY_HASH_FILE }} + CURRENT_HASH=${{ github.sha }} + if [ -f $HASH_FILE ]; then + PREV_HASH=$(cat $HASH_FILE) + asv continuous $PREV_HASH $CURRENT_HASH --verbose || true + asv compare $PREV_HASH $CURRENT_HASH --sort ratio --verbose + fi + echo $CURRENT_HASH > $HASH_FILE + + - name: Update last nightly hash in cache + uses: actions/cache@v4 + with: + path: ${{ env.WORKING_DIR }} + key: nightly-results-${{ steps.nightly-dates.outputs.today }} \ No newline at end of file diff --git a/.github/workflows/asv-pr.yml b/.github/workflows/asv-pr.yml new file mode 100644 index 0000000..bf5aed6 --- /dev/null +++ b/.github/workflows/asv-pr.yml @@ -0,0 +1,86 @@ +# This workflow will run benchmarks with airspeed velocity (asv) for pull requests. +# It will compare the performance of the main branch with the performance of the merge +# with the new changes. It then publishes a comment with this assessment by triggering +# the publish-benchmarks-pr workflow. +# Based on https://securitylab.github.com/research/github-actions-preventing-pwn-requests/. +name: Run benchmarks for PR + +on: + pull_request: + branches: [ main ] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PYTHON_VERSION: "3.10" + WORKING_DIR: ${{ github.workspace }}/benchmarks + ARTIFACTS_DIR: ${{ github.workspace }}/artifacts + +jobs: + setup-python: + runs-on: ubuntu-latest + steps: + - name: Cache Python ${{ env.PYTHON_VERSION }} + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-${{ env.PYTHON_VERSION }} + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + asv-pr: + runs-on: ubuntu-latest + needs: setup-python + defaults: + run: + working-directory: ${{ env.WORKING_DIR }} + steps: + - name: Checkout PR branch of the repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Display Workflow Run Information + run: | + echo "Workflow Run ID: ${{ github.run_id }}" + - name: Cache Python ${{ env.PYTHON_VERSION }} + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-${{ env.PYTHON_VERSION }} + - name: Install dependencies + run: | + sudo apt-get update + python -m pip install --upgrade pip + pip install asv==0.6.1 virtualenv tabulate lf-asv-formatter + - name: Make artifacts directory + run: mkdir -p ${{ env.ARTIFACTS_DIR }} + - name: Save pull request number + run: echo ${{ github.event.pull_request.number }} > ${{ env.ARTIFACTS_DIR }}/pr + - name: Get current job logs URL + uses: Tiryoh/gha-jobid-action@v1 + id: jobs + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + job_name: ${{ github.job }} + - name: Create ASV machine config file + run: asv machine --machine gh-runner --yes + - name: Save comparison of PR against main branch + run: | + git remote add upstream https://github.com/${{ github.repository }}.git + git fetch upstream + asv continuous upstream/main HEAD --verbose || true + asv compare upstream/main HEAD --sort ratio --verbose | tee output + python -m lf_asv_formatter --asv_version "$(echo asv --version)" + printf "\n\nClick [here]($STEP_URL) to view all benchmarks." >> output + mv output ${{ env.ARTIFACTS_DIR }} + env: + STEP_URL: "${{ steps.jobs.outputs.html_url }}#step:11:1" + - name: Upload artifacts (PR number and benchmarks output) + uses: actions/upload-artifact@v4 + with: + name: benchmark-artifacts + path: ${{ env.ARTIFACTS_DIR }} \ No newline at end of file diff --git a/.github/workflows/build-documentation.yml b/.github/workflows/build-documentation.yml index e329f57..638e7b6 100644 --- a/.github/workflows/build-documentation.yml +++ b/.github/workflows/build-documentation.yml @@ -9,15 +9,19 @@ on: pull_request: branches: [ main ] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.10 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install dependencies diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml deleted file mode 100644 index dfcf56b..0000000 --- a/.github/workflows/linting.yml +++ /dev/null @@ -1,36 +0,0 @@ -# This workflow will install Python dependencies, then perform static linting analysis. -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Lint - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.8', '3.9', '3.10'] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - python -m pip install --upgrade pip - pip install . - pip install .[dev] - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Analyze code with linter - - uses: psf/black@stable - with: - src: ./src diff --git a/.github/workflows/pre-commit-ci.yml b/.github/workflows/pre-commit-ci.yml new file mode 100644 index 0000000..a57e221 --- /dev/null +++ b/.github/workflows/pre-commit-ci.yml @@ -0,0 +1,35 @@ +# This workflow runs pre-commit hooks on pushes and pull requests to main +# to enforce coding style. To ensure correct configuration, please refer to: +# https://lincc-ppt.readthedocs.io/en/latest/practices/ci_precommit.html +name: Run pre-commit hooks + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + pre-commit-ci: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install dependencies + run: | + sudo apt-get update + python -m pip install --upgrade pip + pip install .[dev] + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - uses: pre-commit/action@v3.0.1 + with: + extra_args: --all-files --verbose + env: + SKIP: "check-lincc-frameworks-template-version,no-commit-to-branch,check-added-large-files,validate-pyproject,sphinx-build,pytest-check" + - uses: pre-commit-ci/lite-action@v1.0.2 + if: failure() && github.event_name == 'pull_request' && github.event.pull_request.draft == false \ No newline at end of file diff --git a/.github/workflows/publish-benchmarks-pr.yml b/.github/workflows/publish-benchmarks-pr.yml new file mode 100644 index 0000000..45ed928 --- /dev/null +++ b/.github/workflows/publish-benchmarks-pr.yml @@ -0,0 +1,53 @@ +# This workflow publishes a benchmarks comment on a pull request. It is triggered after the +# benchmarks are computed in the asv-pr workflow. This separation of concerns allows us limit +# access to the target repository private tokens and secrets, increasing the level of security. +# Based on https://securitylab.github.com/research/github-actions-preventing-pwn-requests/. +name: Publish benchmarks comment to PR + +on: + workflow_run: + workflows: ["Run benchmarks for PR"] + types: [completed] + +jobs: + upload-pr-comment: + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + permissions: + issues: write + pull-requests: write + steps: + - name: Display Workflow Run Information + run: | + echo "Workflow Run ID: ${{ github.event.workflow_run.id }}" + echo "Head SHA: ${{ github.event.workflow_run.head_sha }}" + echo "Head Branch: ${{ github.event.workflow_run.head_branch }}" + echo "Conclusion: ${{ github.event.workflow_run.conclusion }}" + echo "Event: ${{ github.event.workflow_run.event }}" + - name: Download artifact + uses: dawidd6/action-download-artifact@v3 + with: + name: benchmark-artifacts + run_id: ${{ github.event.workflow_run.id }} + - name: Extract artifacts information + id: pr-info + run: | + printf "PR number: $(cat pr)\n" + printf "Output:\n$(cat output)" + printf "pr=$(cat pr)" >> $GITHUB_OUTPUT + - name: Find benchmarks comment + uses: peter-evans/find-comment@v3 + id: find-comment + with: + issue-number: ${{ steps.pr-info.outputs.pr }} + comment-author: 'github-actions[bot]' + body-includes: view all benchmarks + - name: Create or update benchmarks comment + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ steps.pr-info.outputs.pr }} + body-path: output + edit-mode: replace \ No newline at end of file diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 5367eb6..f7cecc2 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -1,5 +1,5 @@ # This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries +# For more information see: https://github.com/pypa/gh-action-pypi-publish#trusted-publishing # This workflow uses actions that are not certified by GitHub. # They are provided by a third-party and are governed by @@ -19,11 +19,12 @@ jobs: deploy: runs-on: ubuntu-latest - + permissions: + id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install dependencies @@ -33,7 +34,4 @@ jobs: - name: Build package run: python -m build - name: Publish package - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 928fdca..3107560 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -6,8 +6,13 @@ name: Unit test smoke test on: + + # Runs this workflow automatically schedule: - cron: 45 6 * * * + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: jobs: build: @@ -15,24 +20,23 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10'] + python-version: ['3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | sudo apt-get update python -m pip install --upgrade pip - pip install . - pip install .[dev] + pip install -e .[dev] if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: List dependencies run: | pip list - name: Run unit tests with pytest run: | - python -m pytest tests + python -m pytest \ No newline at end of file diff --git a/.github/workflows/testing-and-coverage.yml b/.github/workflows/testing-and-coverage.yml index 8b5fd56..1654e60 100644 --- a/.github/workflows/testing-and-coverage.yml +++ b/.github/workflows/testing-and-coverage.yml @@ -15,23 +15,24 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10'] + python-version: ['3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | sudo apt-get update python -m pip install --upgrade pip - pip install . - pip install .[dev] + pip install -e .[dev] if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Run unit tests with pytest run: | - python -m pytest tests --cov=koffi --cov-report=xml + python -m pytest --cov=koffi --cov-report=xml - name: Upload coverage report to codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 6f294ff..e6cce4e 100644 --- a/.gitignore +++ b/.gitignore @@ -140,4 +140,15 @@ dask-worker-space/ tmp/ # Mac OS -.DS_Store \ No newline at end of file +<<<<<<< before updating +.DS_Store +======= +.DS_Store + +# Airspeed Velocity performance results +_results/ +_html/ + +# Project initialization script +.initialize_new_project.sh +>>>>>>> after updating diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d08084b..d01dac1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,4 @@ repos: - # Compare the local template version to the latest remote template version # This hook should always pass. It will print a message if the local version # is out of date. @@ -10,7 +9,6 @@ repos: name: Check template version description: Compare current template version against latest verbose: true - # Clear output from jupyter notebooks so that only the input cells are committed. - repo: local hooks: @@ -21,23 +19,7 @@ repos: stages: [commit] language: system entry: jupyter nbconvert --clear-output - - # Run unit tests, verify that they pass. Note that coverage is run against - # the ./src directory here because that is what will be committed. In the - # github workflow script, the coverage is run against the installed package - # and uploaded to Codecov by calling pytest like so: - # `python -m pytest --cov= --cov-report=xml` - - repo: local - hooks: - - id: pytest-check - name: Run unit tests - description: Run unit tests with pytest. - entry: bash -c "if python -m pytest --co -qq; then python -m pytest --cov=./src --cov-report=html; fi" - language: system - pass_filenames: false - always_run: true - - # prevents committing directly branches named 'main' and 'master'. + # Prevents committing directly branches named 'main' and 'master'. - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: @@ -48,40 +30,34 @@ repos: name: Check for large files description: Prevent the user from committing very large files. args: ['--maxkb=500'] - - # verify that pyproject.toml is well formed + # Verify that pyproject.toml is well formed - repo: https://github.com/abravalheri/validate-pyproject rev: v0.12.1 hooks: - id: validate-pyproject name: Validate pyproject.toml description: Verify that pyproject.toml adheres to the established schema. - - # Automatically sort the imports used in .py files - - repo: https://github.com/pycqa/isort - rev: 5.12.0 + # Verify that GitHub workflows are well formed + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.28.0 + hooks: + - id: check-github-workflows + args: ["--verbose"] + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.2.1 hooks: - - id: isort - name: isort (python files in src/ and tests/) - description: Sort and organize imports in .py files. - types: [python] - files: ^(src|tests)/ - - - # Analyze the code style and report code that doesn't adhere. - - repo: https://github.com/psf/black - rev: 23.1.0 + - id: ruff + name: Lint code using ruff; sort and organize imports + types_or: [ python, pyi ] + args: ["--fix"] + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.2.1 hooks: - - id: black - types: [python] - files: ^(src|tests)/ - # It is recommended to specify the latest version of Python - # supported by your project here, or alternatively use - # pre-commit's default_language_version, see - # https://pre-commit.com/#top_level-default_language_version - language_version: python3.10 - - + - id: ruff-format + name: Format code using ruff + types_or: [ python, pyi, jupyter ] # Make sure Sphinx can build the documentation while explicitly omitting # notebooks from the docs, so users don't have to wait through the execution # of each notebook or each commit. By default, these will be checked in the @@ -108,3 +84,17 @@ repos: "-D", # Flag to override settings in conf.py "exclude_patterns=notebooks/*", # Exclude our notebooks from pre-commit ] + # Run unit tests, verify that they pass. Note that coverage is run against + # the ./src directory here because that is what will be committed. In the + # github workflow script, the coverage is run against the installed package + # and uploaded to Codecov by calling pytest like so: + # `python -m pytest --cov= --cov-report=xml` + - repo: local + hooks: + - id: pytest-check + name: Run unit tests + description: Run unit tests with pytest. + entry: bash -c "if python -m pytest --co -qq; then python -m pytest --cov=./src --cov-report=html; fi" + language: system + pass_filenames: false + always_run: true diff --git a/.setup_dev.sh b/.setup_dev.sh new file mode 100644 index 0000000..d8cd955 --- /dev/null +++ b/.setup_dev.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# This script should be run by new developers to install this package in +# editable mode and configure their local environment + +echo "Checking virtual environment" +if [ -z "${VIRTUAL_ENV}" ] && [ -z "${CONDA_PREFIX}" ]; then + echo 'No virtual environment detected: none of $VIRTUAL_ENV or $CONDA_PREFIX is set.' + echo + echo "=== This script is going to install the project in the system python environment ===" + echo "Proceed? [y/N]" + read -r RESPONCE + if [ "${RESPONCE}" != "y" ]; then + echo "See https://lincc-ppt.readthedocs.io/ for details." + echo "Exiting." + exit 1 + fi + +fi + +echo "Checking pip version" +MINIMUM_PIP_VERSION=22 +pipversion=( $(python -m pip --version | awk '{print $2}' | sed 's/\./ /g') ) +if let "${pipversion[0]}<${MINIMUM_PIP_VERSION}"; then + echo "Insufficient version of pip found. Requires at least version ${MINIMUM_PIP_VERSION}." + echo "See https://lincc-ppt.readthedocs.io/ for details." + exit 1 +fi + +echo "Installing package and runtime dependencies in local environment" +python -m pip install -e . > /dev/null + +echo "Installing developer dependencies in local environment" +python -m pip install -e .'[dev]' > /dev/null +if [ -f docs/requirements.txt ]; then python -m pip install -r docs/requirements.txt; fi + +echo "Installing pre-commit" +pre-commit install > /dev/null + +####################################################### +# Include any additional configurations below this line +####################################################### diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json new file mode 100644 index 0000000..82e25f4 --- /dev/null +++ b/benchmarks/asv.conf.json @@ -0,0 +1,80 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + // The name of the project being benchmarked. + "project": "koffi", + // The project's homepage. + "project_url": "https://github.com/lincc-frameworks/koffi", + // The URL or local path of the source code repository for the + // project being benchmarked. + "repo": "..", + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "tip" (for mercurial). + "branches": [ + "HEAD" + ], + "install_command": [ + "python -m pip install {wheel_file}" + ], + "build_command": [ + "python -m build --wheel -o {build_cache_dir} {build_dir}" + ], + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + "dvcs": "git", + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "virtualenv", + // the base URL to show a commit for the project. + "show_commit_url": "https://github.com/lincc-frameworks/koffi/commit/", + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": [ + "3.10" + ], + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty + // list indicates to just test against the default (latest) + // version. + "matrix": { + "Cython": [], + "build": [], + "packaging": [] + }, + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks". + "benchmark_dir": ".", + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env". + "env_dir": "env", + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + "results_dir": "_results", + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": "_html", + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + // `asv` will cache wheels of the recent builds in each + // environment, making them faster to install next time. This is + // number of builds to keep, per environment. + "build_cache_size": 8 + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // } +} \ No newline at end of file diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py new file mode 100644 index 0000000..6a61986 --- /dev/null +++ b/benchmarks/benchmarks.py @@ -0,0 +1,16 @@ +"""Two sample benchmarks to compute runtime and memory usage. + +For more information on writing benchmarks: +https://asv.readthedocs.io/en/stable/writing_benchmarks.html.""" + +from koffi import example_benchmarks + + +def time_computation(): + """Time computations are prefixed with 'time'.""" + example_benchmarks.runtime_computation() + + +def mem_list(): + """Memory computations are prefixed with 'mem' or 'peakmem'.""" + return example_benchmarks.memory_computation() diff --git a/docs/conf.py b/docs/conf.py index 0a2cf2a..7e44385 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -6,12 +6,10 @@ import os import sys - -import autoapi from importlib.metadata import version # Define path to the code to be documented **relative to where conf.py (this file) is kept** -sys.path.insert(0, os.path.abspath('../src/')) +sys.path.insert(0, os.path.abspath("../src/")) # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information @@ -31,12 +29,25 @@ extensions.append("autoapi.extension") extensions.append("nbsphinx") -templates_path = [] -exclude_patterns = ['_build', '**.ipynb_checkpoints'] +# -- sphinx-copybutton configuration ---------------------------------------- +extensions.append("sphinx_copybutton") +## sets up the expected prompt text from console blocks, and excludes it from +## the text that goes into the clipboard. +copybutton_exclude = ".linenos, .gp" +copybutton_prompt_text = ">> " -master_doc = "index" # This assumes that sphinx-build is called from the root directory -html_show_sourcelink = False # Remove 'view source code' from top of page (for html, not python) -add_module_names = False # Remove namespaces from class/method signatures +## lets us suppress the copy button on select code blocks. +copybutton_selector = "div:not(.no-copybutton) > div.highlight > pre" + +templates_path = [] +exclude_patterns = ["_build", "**.ipynb_checkpoints"] + +# This assumes that sphinx-build is called from the root directory +master_doc = "index" +# Remove 'view source code' from top of page (for html, not python) +html_show_sourcelink = False +# Remove namespaces from class/method signatures +add_module_names = False autoapi_type = "python" autoapi_dirs = ["../src"] diff --git a/docs/index.rst b/docs/index.rst index 8104343..2b75e26 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,6 +5,43 @@ Welcome to koffi's documentation! ======================================================================================== +Dev Guide - Getting Started +--------------------------- + +Before installing any dependencies or writing code, it's a great idea to create a +virtual environment. LINCC-Frameworks engineers primarily use `conda` to manage virtual +environments. If you have conda installed locally, you can run the following to +create and activate a new environment. + +.. code-block:: console + + >> conda create env -n python=3.10 + >> conda activate + + +Once you have created a new environment, you can install this project for local +development using the following commands: + +.. code-block:: console + + >> pip install -e .'[dev]' + >> pre-commit install + >> conda install pandoc + + +Notes: + +1) The single quotes around ``'[dev]'`` may not be required for your operating system. +2) ``pre-commit install`` will initialize pre-commit for this local repository, so + that a set of tests will be run prior to completing a local commit. For more + information, see the Python Project Template documentation on + `pre-commit `_. +3) Installing ``pandoc`` allows you to verify that automatic rendering of Jupyter notebooks + into documentation for ReadTheDocs works as expected. For more information, see + the Python Project Template documentation on + `Sphinx and Python Notebooks `_. + + .. toctree:: :hidden: diff --git a/docs/notebooks/intro_notebook.ipynb b/docs/notebooks/intro_notebook.ipynb index 2e7779f..0589b29 100644 --- a/docs/notebooks/intro_notebook.ipynb +++ b/docs/notebooks/intro_notebook.ipynb @@ -2,97 +2,70 @@ "cells": [ { "cell_type": "markdown", - "id": "accepting-editor", + "id": "textblock1", "metadata": { "cell_marker": "\"\"\"" }, "source": [ - "# Introducing Jupyter Notebooks\n", + "# Introducing Jupyter Notebooks in Sphinx\n", "\n", - "_(The example used here is JamesALeedham's notebook: [intro.ipynb](https://github.com/JamesALeedham/Sphinx-Autosummary-Recursion/blob/master/docs/notebooks/intro.ipynb))_\n", + "This notebook showcases very basic functionality of rendering your jupyter notebooks as tutorials inside your sphinx documentation.\n", "\n", - "First, set up the environment:" + "As part of the LINCC Frameworks python project template, your notebooks will be executed AND rendered at document build time.\n", + "\n", + "You can read more about Sphinx, ReadTheDocs, and building notebooks in [LINCC's documentation](https://lincc-ppt.readthedocs.io/en/latest/practices/sphinx.html)" ] }, { "cell_type": "code", "execution_count": null, - "id": "actual-thirty", + "id": "codeblock1", "metadata": {}, "outputs": [], "source": [ - "import matplotlib\n", - "import matplotlib.pyplot as pl\n", - "import numpy as np\n", - "\n", - "try:\n", - " from IPython import get_ipython\n", - " get_ipython().run_line_magic('matplotlib', 'inline')\n", - "except AttributeError:\n", - " print('Magic function can only be used in IPython environment')\n", - " matplotlib.use('Agg')\n", - "\n", - "pl.rcParams[\"figure.figsize\"] = [15, 8]" + "def sierpinsky(order):\n", + " \"\"\"Define a method that will create a Sierpinsky triangle of given order,\n", + " and will print it out.\"\"\"\n", + " triangles = [\"*\"]\n", + " for i in range(order):\n", + " spaces = \" \" * (2**i)\n", + " triangles = [spaces + triangle + spaces for triangle in triangles] + [\n", + " triangle + \" \" + triangle for triangle in triangles\n", + " ]\n", + " print(f\"Printing order {order} triangle\")\n", + " print(\"\\n\".join(triangles))" ] }, { "cell_type": "markdown", - "id": "coral-upper", + "id": "textblock2", "metadata": { "cell_marker": "\"\"\"", "lines_to_next_cell": 1 }, "source": [ - "Then, define a function that creates a pretty graph:" + "Then, call our method a few times. This will happen on the fly during notebook rendering." ] }, { "cell_type": "code", "execution_count": null, - "id": "funded-protection", - "metadata": { - "lines_to_next_cell": 1 - }, + "id": "codeblock2", + "metadata": {}, "outputs": [], "source": [ - "def SineAndCosineWaves():\n", - " # Get a large number of X values for a nice smooth curve. Using Pi as np.sin requires radians...\n", - " x = np.linspace(0, 2 * np.pi, 180)\n", - " # Convert radians to degrees to make for a meaningful X axis (1 radian = 57.29* degrees)\n", - " xdeg = 57.29577951308232 * np.array(x)\n", - " # Calculate the sine of each value of X\n", - " y = np.sin(x)\n", - " # Calculate the cosine of each value of X\n", - " z = np.cos(x)\n", - " # Plot the sine wave in blue, using degrees rather than radians on the X axis\n", - " pl.plot(xdeg, y, color='blue', label='Sine wave')\n", - " # Plot the cos wave in green, using degrees rather than radians on the X axis\n", - " pl.plot(xdeg, z, color='green', label='Cosine wave')\n", - " pl.xlabel(\"Degrees\")\n", - " # More sensible X axis values\n", - " pl.xticks(np.arange(0, 361, 45))\n", - " pl.legend()\n", - " pl.show()" - ] - }, - { - "cell_type": "markdown", - "id": "thorough-cutting", - "metadata": { - "cell_marker": "\"\"\"" - }, - "source": [ - "Finally, call that function to display the graph:" + "for order in range(3):\n", + " sierpinsky(order)" ] }, { "cell_type": "code", "execution_count": null, - "id": "imported-uruguay", + "id": "codeblock3", "metadata": {}, "outputs": [], "source": [ - "SineAndCosineWaves()" + "sierpinsky(4)" ] } ], diff --git a/docs/requirements.txt b/docs/requirements.txt index 5c0f7d8..ee05654 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,9 +1,10 @@ -sphinx==6.1.3 -sphinx-rtd-theme==1.2.0 -sphinx-autoapi==2.0.1 -nbsphinx + +ipykernel ipython jupytext -jupyter -matplotlib -numpy +nbconvert +nbsphinx +sphinx +sphinx-autoapi +sphinx-copybutton +sphinx-rtd-theme \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index f0dc46a..c996a4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ classifiers = [ "Programming Language :: Python", ] dynamic = ["version"] +requires-python = ">=3.9" dependencies = [ "ipykernel", # Support for Jupyter notebooks "astropy", @@ -22,24 +23,18 @@ dependencies = [ "tqdm" ] +[project.urls] +"Source Code" = "https://github.com/lincc-frameworks/koffi" + # On a mac, install optional dependencies with `pip install '.[dev]'` (include the single quotes) [project.optional-dependencies] dev = [ + "asv==0.6.1", # Used to compute performance benchmarks + "jupyter", # Clears output from Jupyter notebooks + "pre-commit", # Used to run checks before finalizing a git commit "pytest", "pytest-cov", # Used to report total code coverage - "pre-commit", # Used to run checks before finalizing a git commit - "sphinx==6.1.3", # Used to automatically generate documentation - "sphinx-rtd-theme==1.2.0", # Used to render documentation - "sphinx-autoapi==2.0.1", # Used to automatically generate api documentation - "black", # Used for static linting of files - # if you add dependencies here while experimenting in a notebook and you - # want that notebook to render in your documentation, please add the - # dependencies to ./docs/requirements.txt as well. - "nbconvert", # Needed for pre-commit check to clear output from Python notebooks - "nbsphinx", # Used to integrate Python notebooks into Sphinx documentation - "ipython", # Also used in building notebooks into Sphinx - "matplotlib", # Used in sample notebook intro_notebook.ipynb - "numpy", # Used in sample notebook intro_notebook.ipynb + "ruff", # Used for static linting of files ] [build-system] @@ -59,7 +54,56 @@ testpaths = [ [tool.black] line-length = 110 +target-version = ["py39"] [tool.isort] profile = "black" line_length = 110 + +[tool.ruff] +line-length = 110 +target-version = "py39" + +[tool.ruff.lint] +select = [ + # pycodestyle + "E", + "W", + # Pyflakes + "F", + # pep8-naming + "N", + # pyupgrade + "UP", + # flake8-bugbear + "B", + # flake8-simplify + "SIM", + # isort + "I", + # docstrings + "D101", + "D102", + "D103", + "D106", + "D206", + "D207", + "D208", + "D300", + "D417", + "D419", + +] + +ignore = [ + "UP006", # Allow non standard library generics in type hints + "UP007", # Allow Union in type hints + "SIM114", # Allow if with same arms + "B028", # Allow default warning level + "SIM117", # Allow nested with + "UP015", # Allow redundant open parameters + "UP028", # Allow yield in for loop +] + +[tool.coverage.run] +omit=["src/koffi/_version.py"] diff --git a/src/koffi/example_benchmarks.py b/src/koffi/example_benchmarks.py new file mode 100644 index 0000000..5a77b06 --- /dev/null +++ b/src/koffi/example_benchmarks.py @@ -0,0 +1,14 @@ +"""An example module containing simplistic methods under benchmarking.""" + +import random +import time + + +def runtime_computation(): + """Runtime computation consuming between 0 and 5 seconds.""" + time.sleep(random.uniform(0, 5)) + + +def memory_computation(): + """Memory computation for a random list up to 512 samples.""" + return [0] * random.randint(0, 512) From 476c421da759872b3fa98153b0be7b022def221d Mon Sep 17 00:00:00 2001 From: Max West Date: Thu, 4 Apr 2024 10:57:24 -0700 Subject: [PATCH 2/2] add sphinx depends --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c996a4e..446f235 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,8 @@ dev = [ "pytest", "pytest-cov", # Used to report total code coverage "ruff", # Used for static linting of files + "sphinx", + "sphinx-copybutton", ] [build-system]