fix test data - 2 #67
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci/cd | |
| on: | |
| push: | |
| branches: | |
| - '**' | |
| tags-ignore: | |
| - '**' | |
| jobs: | |
| ########################################################################### | |
| # PACKAGING | |
| ########################################################################### | |
| py-setup: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| - uses: WIPACrepo/[email protected] | |
| with: | |
| python_min: 3.11 | |
| python_max: 3.11 | |
| pypi_name: icecube-simprod-histogram | |
| author: IceCube | |
| author_email: [email protected] | |
| keywords: | | |
| "histogram sampling" simulation statistics | |
| ########################################################################### | |
| # LINTERS | |
| ########################################################################### | |
| py-versions: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.versions.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - id: versions | |
| uses: WIPACrepo/[email protected] | |
| flake8: | |
| needs: [ py-versions ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - uses: WIPACrepo/[email protected] | |
| mypy: | |
| needs: [ py-versions ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - uses: WIPACrepo/[email protected] | |
| ########################################################################### | |
| # FORMATTER | |
| ########################################################################### | |
| code-format: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| - name: Run Ruff for code formatting | |
| run: | | |
| pip install ruff | |
| ruff check --select C408 --fix . --unsafe-fixes | |
| - name: Commit formatted code | |
| run: | | |
| git config user.name github-actions | |
| git config user.email [email protected] | |
| git add . | |
| git commit -m "<bot> auto code format file(s)" || true | |
| git push || true | |
| ########################################################################### | |
| # TESTS | |
| ########################################################################### | |
| unit-tests: | |
| needs: [ py-versions ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - run: | | |
| pip install .[tests] | |
| - name: Run unit tests | |
| run: | | |
| pytest -vvv tests/unit/ | |
| test-wrapper-script: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| num_datasets: | |
| - 1 | |
| - 25 | |
| - 10000000 | |
| base_path: | |
| - /tmp/data/sim/Upgrade/2022/generated/neutrino-generator/88888 | |
| - /tmp/data/sim/IceCube/2023/filtered/CORSIKA | |
| - /tmp/data/sim/Upgrade/2022/filtered | |
| - /tmp/data/sim/IceCube/2023 | |
| - /tmp/data/sim/Upgrade | |
| - /tmp/data/sim | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python environment | |
| uses: actions/setup-python@v4 | |
| - name: Create a mock dataset structure | |
| run: | | |
| set -e | |
| job_range_dpaths=( | |
| /tmp/data/sim/{IceCube,Upgrade}/{2022,2023}/{generated,filtered}/{CORSIKA,neutrino-generator}/{77777,88888,99999}/{00-11,22-33,44-55} | |
| ) | |
| # Create directories and conditionally populate files | |
| for dpath in "${job_range_dpaths[@]}"; do | |
| mkdir -p "$dpath"/histos/ | |
| # create 1-5 pkl files | |
| for i in $( seq 1 "$(( (RANDOM % 5) + 1 ))" ); do | |
| random_file=$(find "tests/data/simprod-histograms" -type f -name "*.pkl" | shuf -n 1) | |
| cp "$random_file" "$dpath/histos/histo_$i.pkl" | |
| done | |
| done | |
| tree /tmp/data/sim/ | |
| - name: Run script with matrix parameters | |
| run: | | |
| set -e | |
| echo "Testing with base_path=${{ matrix.base_path }}, num_datasets=${{ matrix.num_datasets }}" | |
| ./resources/sample-each-dataset.sh ${{ matrix.base_path }} 0.5 ${{ matrix.num_datasets }} | |
| - name: Validate script execution | |
| run: | | |
| set -e | |
| # Check if the expected number of datasets are processed | |
| processed_count=$(find ${{ matrix.base_path }} -name '*.histo.hdf5' | wc -l) | |
| if [[ $processed_count -lt ${{ matrix.num_datasets }}} ]]; then | |
| echo "Script did not process the expected number of datasets!" | |
| exit 1 | |
| fi | |
| echo "All tests passed." | |
| ########################################################################### | |
| # RELEASE | |
| ########################################################################### | |
| release: | |
| if: github.ref == 'refs/heads/main' | |
| needs: [ py-setup, flake8, mypy, code-format, unit-tests, test-wrapper-script ] | |
| runs-on: ubuntu-latest | |
| concurrency: release # prevent any possible race conditions | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # Python-Package Version Bump | |
| - uses: python-semantic-release/[email protected] | |
| id: psr-psr | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| # PyPI Release | |
| - uses: pypa/[email protected] | |
| if: steps.psr-psr.outputs.released == 'true' | |
| with: | |
| password: ${{ secrets.WIPAC_PYPI_TOKEN }} | |
| # GitHub Release | |
| - uses: python-semantic-release/[email protected] | |
| if: steps.psr-psr.outputs.released == 'true' | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} |