add test

.github/workflows/wipac-cicd.yaml

@@ -119,7 +119,7 @@ jobs:
           set -euo pipefail
           pytest -vvv tests/unit/
 
-  test-wrapper-script:
+  test-sample-each-dataset-sh:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -140,14 +140,14 @@ jobs:
         uses: actions/checkout@v3
       - name: Set up Python environment
         uses: actions/setup-python@v4
-      - name: Create a mock dataset structure
+
+      - name: Create source dataset dirs/files
         run: |
           set -euo pipefail
           job_range_dpaths=(
             /tmp/data/sim/{IceCube,Upgrade}/{2022,2023}/{generated,filtered}/{CORSIKA,neutrino-generator}/{77777,88888,99999}/{00-11,22-33,44-55}
           )
 
-          # Create directories and conditionally populate files
           for dpath in "${job_range_dpaths[@]}"; do
             mkdir -p "$dpath"/histos/
             # create 1-5 pkl files
@@ -162,7 +162,7 @@ jobs:
           set -euo pipefail
           tree /tmp/data/sim/
 
-      - name: Run script with matrix parameters
+      - name: Run script
         run: |
           set -euo pipefail
           set -x
@@ -197,13 +197,134 @@ jobs:
           set -euo pipefail
           tree /tmp/data/sim/
 
+  test-cp-dataset-histos-sh:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        prev_histos_setting:
+          - none
+          - overwrite
+          - keep
+        base_path:
+          - /tmp/data/sim/IceCube/2023/generated/neutrino-generator
+          - /tmp/data/sim/Upgrade/2022/filtered/CORSIKA
+        dest_dir:
+          - /tmp/mycopy
+    env:
+      OLD_FILE_MODTIME: 0
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Set up Python environment
+        uses: actions/setup-python@v4
+
+      - name: Create source dataset dirs/files
+        run: |
+          set -euo pipefail
+          dataset_dpaths=(
+            /tmp/data/sim/{IceCube,Upgrade}/{2022,2023}/{generated,filtered}/{CORSIKA,neutrino-generator}/{77777,88888,99999}/
+          )
+
+          for dpath in "${dataset_dpaths[@]}"; do
+            histo="$dpath"/"$(basename $dpath).histo.hdf5"
+            mkdir $(dirname $histo)
+            touch $histo
+            # pre-create some of these files in the destination
+            if [[ "${{ matrix.prev_histos_setting }}" == "overwrite" || "${{ matrix.prev_histos_setting }}" == "keep" ]]; then
+              if (( RANDOM % 100 < 25 )); then
+                relative_path="${dpath#*/sim/}"
+                dest_dataset_dir="${{ matrix.dest_dir }}/sim/$relative_path"
+                touch "$dest_dataset_dir"/"$(basename "$dest_dataset_dir").histo.hdf5"
+              fi
+            fi
+          done
+          
+          # set the oldest file's mod time
+          if [[ "${{ matrix.prev_histos_setting }}" == "overwrite" || "${{ matrix.prev_histos_setting }}" == "keep" ]]; then
+            oldest_modtime=$(find "${{ matrix.dest_dir }}" -name "*.histo.hdf5" -type f -exec stat --format='%Y' {} + | sort -n | head -1)
+            echo "OLD_FILE_MODTIME=$oldest_modtime" >> $GITHUB_ENV
+          fi
+
+      - name: Look at src filetree (before)
+        run: |
+          set -euo pipefail
+          tree /tmp/data/sim/
+
+      - name: Look at dest filetree (before)
+        run: |
+          set -euo pipefail
+          tree ${{ matrix.dest_dir }}
+
+      - name: Run script
+        run: |
+          set -euo pipefail
+          prev_histos_setting=""
+          if [[ "${{ matrix.prev_histos_setting }}" == "overwrite" ]]; then
+            force_flag="--force"
+          fi
+          set -x
+          ./cp-dataset-histos.sh ${{ matrix.base_path }} ${{ matrix.dest_dir }} $force_flag
+
+      - name: Validate copied histograms
+        run: |
+          set -euo pipefail
+
+          src_count=$(find ${{ matrix.base_path }} -name "*.histo.hdf5" | wc -l)
+          dest_count=$(find ${{ matrix.dest_dir }} -name "*.histo.hdf5" | wc -l)
+          echo "Source histograms: $src_count"
+          echo "Copied histograms: $dest_count"
+          if [[ $src_count -ne $dest_count ]]; then
+            echo "Copied histograms count ($dest_count) does not match source histograms count ($src_count)!"
+            exit 1
+          fi
+          
+          # check the overwriting settings
+          oldest_modtime=$(find "${{ matrix.dest_dir }}" -name "*.histo.hdf5" -type f -exec stat --format='%Y' {} + | sort -n | head -1)
+          echo "Oldest histo file modtime: $oldest_modtime"
+          case "${{ matrix.prev_histos_setting }}" in
+            none)
+              # oldest modtime should be younger than previously-stored value
+              if [[ ! $oldest_modtime -lt $OLD_FILE_MODTIME ]]; then
+                echo "ERROR: there is an older file in here!" >&2
+                exit 1
+              fi
+              ;;
+            overwrite)
+              # oldest modtime should be younger than previously-stored value
+              if [[ ! $oldest_modtime -lt $OLD_FILE_MODTIME ]]; then
+                echo "ERROR: there is an older file in here! aka script didn't overwrite" >&2
+                exit 1
+              fi
+              ;;
+            keep)
+              # oldest modtime should be the previously-stored value
+              if [[ $oldest_modtime -ne $OLD_FILE_MODTIME ]]; then
+                echo "ERROR: there is no older file in here! aka the scrip did overwrite" >&2
+                exit 1
+              fi
+              ;;
+            *)
+              echo "Error: Unknown value for prev_histos_setting: $prev_histos_setting" >&2
+              exit 1
+              ;;
+          esac
+
+          echo "All tests passed for base_path=${{ matrix.base_path }} and dest_dir=${{ matrix.dest_dir }}."
+
+      - name: Look at dest filetree (after)
+        run: |
+          set -euo pipefail
+          tree ${{ matrix.dest_dir }}
+
+
   ###########################################################################
   # RELEASE
   ###########################################################################
 
   release:
     if: github.ref == 'refs/heads/main'
-    needs: [ py-setup, flake8, mypy, code-format, unit-tests, test-wrapper-script ]
+    needs: [ py-setup, flake8, mypy, code-format, unit-tests, test-sample-each-dataset-sh, test-cp-dataset-histos-sh ]
     runs-on: ubuntu-latest
     concurrency: release  # prevent any possible race conditions
     steps:

scripts/cp-dataset-histos.sh

@@ -69,7 +69,7 @@ cp_histo() {
     #   src_dataset_dir  = '.../sim/IceCube/2023/generated/neutrino-generator/22645'
     #   relative_path    = 'IceCube/2023/generated/neutrino-generator/22645'
     #   dest_dataset_dir = '$DEST_DIR/sim/IceCube/2023/generated/neutrino-generator/22645'
-    local relative_path="${src_dataset_dir#"$SIM"/}"
+    local relative_path="${src_dataset_dir#*"$SIM"/}"
     local dest_dataset_dir="$DEST_DIR/$SIM/$relative_path"
     mkdir -p "$dest_dataset_dir"