ci/cd

update count logic - 2 #71

Workflow file for this run

.github/workflows/wipac-cicd.yaml at c74796e

	name: ci/cd

	on:
	push:
	branches:
	- '**'
	tags-ignore:
	- '**'

	jobs:

	###########################################################################
	# PACKAGING
	###########################################################################

	py-setup:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
	- uses: WIPACrepo/[email protected]
	with:
	python_min: 3.11
	python_max: 3.11
	pypi_name: icecube-simprod-histogram
	author: IceCube
	author_email: [email protected]
	keywords: \|
	"histogram sampling" simulation statistics


	###########################################################################
	# LINTERS
	###########################################################################

	py-versions:
	runs-on: ubuntu-latest
	outputs:
	matrix: ${{ steps.versions.outputs.matrix }}
	steps:
	- uses: actions/checkout@v3
	- id: versions
	uses: WIPACrepo/[email protected]

	flake8:
	needs: [ py-versions ]
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
	steps:
	- uses: actions/checkout@v3
	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.py3 }}
	- uses: WIPACrepo/[email protected]

	mypy:
	needs: [ py-versions ]
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
	steps:
	- uses: actions/checkout@v3
	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.py3 }}
	- uses: WIPACrepo/[email protected]

	###########################################################################
	# FORMATTER
	###########################################################################

	code-format:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- uses: actions/setup-python@v4
	- name: Run Ruff for code formatting
	run: \|
	set -euo pipefail
	pip install ruff
	ruff check --select C408 --fix . --unsafe-fixes
	- name: Commit formatted code
	run: \|
	set -euo pipefail
	git config user.name github-actions
	git config user.email [email protected]
	git add .
	git commit -m "<bot> auto code format file(s)" \|\| true
	git push \|\| true

	###########################################################################
	# TESTS
	###########################################################################

	unit-tests:
	needs: [ py-versions ]
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
	steps:
	- uses: actions/checkout@v3
	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.py3 }}
	- name: install
	run: \|
	set -euo pipefail
	pip install .[tests]
	- name: Run unit tests
	run: \|
	set -euo pipefail
	pytest -vvv tests/unit/

	test-wrapper-script:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	max_num_datasets:
	- 1
	- 25
	- 10000000
	base_path:
	- /tmp/data/sim/Upgrade/2022/generated/neutrino-generator/88888
	- /tmp/data/sim/IceCube/2023/filtered/CORSIKA
	- /tmp/data/sim/Upgrade/2022/filtered
	- /tmp/data/sim/IceCube/2023
	- /tmp/data/sim/Upgrade
	- /tmp/data/sim
	steps:
	- name: Checkout repository
	uses: actions/checkout@v3
	- name: Set up Python environment
	uses: actions/setup-python@v4
	- name: Create a mock dataset structure
	run: \|
	set -euo pipefail
	job_range_dpaths=(
	/tmp/data/sim/{IceCube,Upgrade}/{2022,2023}/{generated,filtered}/{CORSIKA,neutrino-generator}/{77777,88888,99999}/{00-11,22-33,44-55}
	)

	# Create directories and conditionally populate files
	for dpath in "${job_range_dpaths[@]}"; do
	mkdir -p "$dpath"/histos/
	# create 1-5 pkl files
	for i in $( seq 1 "$(( (RANDOM % 5) + 1 ))" ); do
	random_file=$(find "tests/data/simprod-histograms" -type f -name "*.pkl" \| shuf -n 1)
	cp "$random_file" "$dpath/histos/histo_$i.pkl"
	done
	done
	tree /tmp/data/sim/

	- name: Run script with matrix parameters
	run: \|
	set -euo pipefail
	set -x
	./resources/sample-each-dataset.sh ${{ matrix.base_path }} 0.5 ${{ matrix.max_num_datasets }}

	- name: Validate script execution
	run: \|
	set -euo pipefail
	# Count dataset directories containing at least one "*.histo.hdf5" file
	available_datasets=$(find ${{ matrix.base_path }} -type d -regex ".*/[0-9]+-[0-9]+$" -exec dirname {} \; \| sort -u \| wc -l)
	echo "Available datasets: $available_datasets"

	# Use the lesser of available_datasets and num_datasets for validation
	effective_num_datasets=$(( available_datasets < ${{ matrix.max_num_datasets }} ? available_datasets : ${{ matrix.max_num_datasets }} ))
	echo "Expected datasets: $effective_num_datasets"

	# Check processed count
	processed_count=$(find ${{ matrix.base_path }} -name '*.histo.hdf5' \| wc -l)
	echo "Processed count: $processed_count"

	if [[ $processed_count -lt $effective_num_datasets ]]; then
	echo "Script did not process the expected number of datasets!"
	exit 1
	fi

	echo "All tests passed."

	###########################################################################
	# RELEASE
	###########################################################################

	release:
	if: github.ref == 'refs/heads/main'
	needs: [ py-setup, flake8, mypy, code-format, unit-tests, test-wrapper-script ]
	runs-on: ubuntu-latest
	concurrency: release # prevent any possible race conditions
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	# Python-Package Version Bump
	- uses: python-semantic-release/[email protected]
	id: psr-psr
	with:
	github_token: ${{ secrets.GITHUB_TOKEN }}
	# PyPI Release
	- uses: pypa/[email protected]
	if: steps.psr-psr.outputs.released == 'true'
	with:
	password: ${{ secrets.WIPAC_PYPI_TOKEN }}
	# GitHub Release
	- uses: python-semantic-release/[email protected]
	if: steps.psr-psr.outputs.released == 'true'
	with:
	github_token: ${{ secrets.GITHUB_TOKEN }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

update count logic - 2 #71

Workflow file

update count logic - 2 #71

Jobs

Run details

Workflow file for this run