Merge pull request #5 from icecube/UpdateSetupAndCI

Add pre-commit hooks, update installation via `pyproject.toml`, update manual tests

.coveragerc

@@ -0,0 +1,9 @@
+# .coveragerc
+
+[run]
+branch = True
+source = dnn_reco
+
+[report]
+
+omit = dnn_reco/__about__.py

.github/workflows/test_suite.yml

@@ -0,0 +1,46 @@
+name: Unit Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [
+          "3.8",
+          "3.9",
+          "3.10",
+          "3.11",
+          "3.12",
+        ]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: pre-commit/[email protected]
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[test]
+      - name: Test with pytest
+        run: |
+          coverage run -m pytest  -v -s
+      - name: Generate Coverage Report
+        run: |
+          coverage report -m
+        continue-on-error: false
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: icecube/dnn_reco

.gitignore

@@ -15,4 +15,3 @@ tmp/*
 !logs/.gitkeep
 !checkpoints/.gitkeep
 !configs/user/.gitkeep
-

.pre-commit-config.yaml

@@ -0,0 +1,72 @@
+# Adapted from simweights
+# SPDX-FileCopyrightText: © 2022 the SimWeights contributors
+#
+# SPDX-License-Identifier: BSD-2-Clause
+
+ci:
+  autoupdate_commit_msg: autoupdate pre-commit hooks
+  autoupdate_schedule: quarterly
+repos:
+  - repo: https://github.com/psf/black
+    rev: 24.3.0
+    hooks:
+      - id: black
+        exclude: ^tests_manual/test_data/.*
+      - id: black-jupyter
+        exclude: ^tests_manual/test_data/.*
+  - repo: https://github.com/asottile/blacken-docs
+    rev: "1.16.0"
+    hooks:
+      - id: blacken-docs
+        args: [-l 100]
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.5
+    hooks:
+      - id: ruff
+        args: [--fix, --show-fixes]
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        args: [--toml, pyproject.toml]
+        additional_dependencies:
+          - tomli
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0
+    hooks:
+      - id: python-no-log-warn
+      - id: python-no-eval
+        exclude: ^tests/.*
+      - id: python-use-type-annotations
+      - id: rst-backticks
+      - id: rst-directive-colons
+      - id: rst-inline-touching-normal
+  - repo: https://github.com/Lucas-C/pre-commit-hooks
+    rev: v1.5.5
+    hooks:
+      - id: forbid-crlf
+      - id: forbid-tabs
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-added-large-files
+        exclude: ^tests_manual/test_data/.*
+      - id: check-ast
+      - id: check-builtin-literals
+      - id: check-case-conflict
+      - id: check-docstring-first
+      - id: check-executables-have-shebangs
+      - id: check-json
+      - id: check-merge-conflict
+      - id: check-shebang-scripts-are-executable
+      - id: check-toml
+      - id: check-vcs-permalinks
+      - id: debug-statements
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: fix-byte-order-marker
+      - id: mixed-line-ending
+      - id: name-tests-test
+        args: [--pytest-test-first]
+      - id: trailing-whitespace
+        exclude: ^(tests_manual/test_data/.*|\.svg$)

CONTRIBUTING.md

@@ -0,0 +1,24 @@
+# Contributing
+
+Welcome and thanks for considering to contribute to this repository!
+
+## Pre-commit Hooks
+
+When contributing to this project, please utilize the pre-commit hooks. If not installed yet, you will need to add the python package `pre-commit`:
+
+    pip install pre-commit
+
+Once the package is installed, simply install the pre-commit hooks defined in the repository by executing:
+
+    pre-commit install
+
+from within the repository directory.
+
+The pre-commit hooks will now automatically run when invoking `git commit`. Note, however, that this requires an active shell that has `pre-commit` installed.
+You can also manually run the pre-commit on single files or on all files via:
+
+    pre-commit run --all-files
+
+If you need to commit something even though there are errors (this should not have to be done!), then you can add the flag `--no-verify` to the `git commit` command. This will bypass the pre-commit hooks.
+
+Additional information is provided here: https://pre-commit.com/

README.md

@@ -1,6 +1,6 @@
-| Documentation | DOI     |
-| :-----------: | :-----: |
-| [![](https://img.shields.io/badge/docs-stable-blue.svg)](https://user-web.icecube.wisc.edu/~mhuennefeld/docs/dnn_reco/html/) | [![DOI](https://zenodo.org/badge/144996337.svg)](https://zenodo.org/badge/latestdoi/144996337) |
+| Testing | Coverage | Documentation | DOI     |
+| :-----: | :------: | :-----------: | :-----: |
+| [![Unit Tests](https://github.com/icecube/dnn_reco/actions/workflows/test_suite.yml/badge.svg)](https://github.com/icecube/dnn_reco/actions/workflows/test_suite.yml) | [![codecov](https://codecov.io/github/icecube/dnn_reco/graph/badge.svg?token=ZQ6K8V9F4U)](https://codecov.io/github/icecube/dnn_reco) | [![](https://img.shields.io/badge/docs-stable-blue.svg)](https://user-web.icecube.wisc.edu/~mhuennefeld/docs/dnn_reco/html/) | [![DOI](https://zenodo.org/badge/144996337.svg)](https://zenodo.org/badge/latestdoi/144996337) |
 
 # dnn_reco
 IceCube DNN reconstruction
@@ -17,17 +17,15 @@ IceCube DNN reconstruction
     # Create data transformation model:
     # We must first create a transformation model that will take care of data normalization and transformation
     python create_trafo_model.py /PATH/TO/MY/YAML/CONFIG/FILE
-    
+
     # Train model:
     # This step can be run with as many config files and settings as you wish.
     # The settings and number of training iterations is automatically logged and will be exported together
     # with the final model.
     python train_model.py /PATH/TO/MY/YAML/CONFIG/FILE
-    
+
     # Export model:
     # Once the model is trained, we can export it, so that it can be used to reconstruct IceCube events with the provided I3Module
     python export_model.py /PATH/TO/MY/YAML/CONFIG/FILE -s /PATH/TO/CONFIG/FILE/USED/TO/CREATE/TRAINING/DATA -o OUTPUT/Directory
 
     # More documentation can be found here: https://icecube.wisc.edu/~mhuennefeld/docs/dnn_reco/html/
-
-

configs/biased_selection/biased_muongun.yaml

@@ -162,7 +162,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/cascade_monopod_cascade_dir_big_kernel.yaml

@@ -278,7 +278,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/cascade_monopod_starting_cascades_big_kernel.yaml

@@ -278,7 +278,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/cascade_monopod_starting_events.yaml

@@ -278,7 +278,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/cascade_monopod_starting_events_big_kernel.yaml

@@ -278,7 +278,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/cascade_monopod_starting_pid_big_kernel.yaml

@@ -278,7 +278,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/dnn_cscd_l3a_starting_events.yaml

@@ -306,7 +306,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/dnn_cscd_l3b/dnn_cscd_l3b_cut2_starting_events_300m_fast_medium.yaml

@@ -202,7 +202,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/dnn_cscd_l3b/dnn_cscd_l3b_cut2_starting_events_300m_fast_small.yaml

@@ -202,7 +202,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/dnn_cscd_l3b_starting_cascades_big_kernel.yaml

@@ -91,7 +91,7 @@
     # Additional MuonGun at CscdL3
     '/net/big-tank/POOL/users/mhuennefeld/data/event_selection/training_data/dnn_cscd_l3b/MuonGun/12*/0.99_eff/00*000-00*999/*.hdf5',
 
-    # Manuel Silva's MuonGun
+    # Manuel's MuonGun
     '/net/big-tank/POOL/users/mhuennefeld/data/event_selection/training_data/dnn_cscd_l3b/MuonGun/2131*/00*000-00*999/*.hdf5',
 
     # Additional MuonGun biased simulation: corner clippers
@@ -177,7 +177,7 @@
     # # Additional MuonGun at CscdL3
     # '/net/big-tank/POOL/users/mhuennefeld/data/event_selection/training_data/dnn_cscd_l3b/MuonGun/12*/0.99_eff/00*000-00*999/*.hdf5',
 
-    # # Manuel Silva's MuonGun
+    # # Manuel's MuonGun
     # '/net/big-tank/POOL/users/mhuennefeld/data/event_selection/training_data/dnn_cscd_l3b/MuonGun/2131*/00*000-00*999/*.hdf5',
 
     # # Additional MuonGun biased simulation: corner clippers
@@ -323,7 +323,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/dnn_cscd_l3b_starting_cascades_big_kernel_strict.yaml

@@ -91,7 +91,7 @@
     # Additional MuonGun at CscdL3
     '/net/big-tank/POOL/users/mhuennefeld/data/event_selection/training_data/dnn_cscd_l3b/MuonGun/12*/0.99_eff/00*000-00*999/*.hdf5',
 
-    # Manuel Silva's MuonGun
+    # Manuel's MuonGun
     '/net/big-tank/POOL/users/mhuennefeld/data/event_selection/training_data/dnn_cscd_l3b/MuonGun/2131*/00*000-00*999/*.hdf5',
 
     # Additional MuonGun biased simulation: corner clippers
@@ -177,7 +177,7 @@
     # # Additional MuonGun at CscdL3
     # '/net/big-tank/POOL/users/mhuennefeld/data/event_selection/training_data/dnn_cscd_l3b/MuonGun/12*/0.99_eff/00*000-00*999/*.hdf5',
 
-    # # Manuel Silva's MuonGun
+    # # Manuel's MuonGun
     # '/net/big-tank/POOL/users/mhuennefeld/data/event_selection/training_data/dnn_cscd_l3b/MuonGun/2131*/00*000-00*999/*.hdf5',
 
     # # Additional MuonGun biased simulation: corner clippers
@@ -323,7 +323,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/dnn_cscd_l3b_starting_events.yaml

@@ -306,7 +306,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/dnn_cscd_l3b_starting_events_big_kernel.yaml

@@ -91,7 +91,7 @@
     # Additional MuonGun at CscdL3
     '/net/big-tank/POOL/users/mhuennefeld/data/event_selection/training_data/dnn_cscd_l3b/MuonGun/12*/0.99_eff/00*000-00*999/*.hdf5',
 
-    # Manuel Silva's MuonGun
+    # Manuel's MuonGun
     '/net/big-tank/POOL/users/mhuennefeld/data/event_selection/training_data/dnn_cscd_l3b/MuonGun/2131*/00*000-00*999/*.hdf5',
 
     # Additional MuonGun biased simulation: corner clippers
@@ -177,7 +177,7 @@
     # # Additional MuonGun at CscdL3
     # '/net/big-tank/POOL/users/mhuennefeld/data/event_selection/training_data/dnn_cscd_l3b/MuonGun/12*/0.99_eff/00*000-00*999/*.hdf5',
 
-    # # Manuel Silva's MuonGun
+    # # Manuel's MuonGun
     # '/net/big-tank/POOL/users/mhuennefeld/data/event_selection/training_data/dnn_cscd_l3b/MuonGun/2131*/00*000-00*999/*.hdf5',
 
     # # Additional MuonGun biased simulation: corner clippers
@@ -323,7 +323,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/dnn_cscd_l3b_starting_events_default_model.yaml

@@ -306,7 +306,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/dnn_cscd_l3b_starting_events_default_model_unweighted.yaml

@@ -306,7 +306,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/dnn_cscd_l3b_starting_events_unweighted.yaml

@@ -306,7 +306,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.

configs/event_selection/dnn_cscd_l3c/dnn_cscd_l3c_cut2_starting_events_300m_fast_medium.yaml

@@ -246,7 +246,7 @@
 # ----------------
 # Biased Selection
 # ----------------
-# Filter events based on the curent reconstruction performance on these
+# Filter events based on the current reconstruction performance on these
 # by defining key, value pairs and a biased fraction.
 # Events will be put in queues, such that biased_fraction of all events
 # passes one of the defined cuts.