diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml new file mode 100644 index 00000000..66e30ccd --- /dev/null +++ b/.JuliaFormatter.toml @@ -0,0 +1,2 @@ +style = "blue" +indent = 4 \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..3a32e6ce --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,5 @@ +FROM mcr.microsoft.com/devcontainers/base:debian + +RUN apt-get update +RUN apt-get install -y software-properties-common +RUN apt-get install -y gdal-bin libgdal-dev libgtk-3-dev graphviz \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..fd4f0b04 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,38 @@ +{ + "name": "Ice Floe Tracker Pipeline", + + "build": { + "dockerfile": "Dockerfile" + }, + + "features": { + "ghcr.io/julialang/devcontainer-features/julia:1": { + "channel": "lts" + }, + "ghcr.io/devcontainers/features/python:1": { + "installTools": true, + "version": "3.11" + }, + "ghcr.io/devcontainers-contrib/features/pipx-package:1": { + "includeDeps": true, + "package": "cylc-rose", + "interpreter": "python3.11" + }, + "ghcr.io/devcontainers/features/docker-in-docker:2": {} + + }, + + "customizations": { + "vscode": { + "extensions": [ + "ms-azuretools.vscode-docker", + "github.vscode-github-actions", + "mechatroner.rainbow-csv", + "analytic-signal.preview-tiff", + "cylc.vscode-cylc" + ] + } + }, + + "postCreateCommand": "" +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..e69de29b diff --git a/.github/workflows/build-docker-julia.yml b/.github/workflows/build-docker-julia.yml index 1a74161f..cf9a0728 100644 --- a/.github/workflows/build-docker-julia.yml +++ b/.github/workflows/build-docker-julia.yml @@ -7,14 +7,13 @@ on: - 'main' tags: - 'v*.*.*' - pull_request: - branches: - - 'main' + pull_request: workflow_dispatch: env: REGISTRY: hub.docker.com IMAGE_NAME: brownccv/icefloetracker-julia + TEST_TAG: brownccv/icefloetracker-julia:test jobs: build_and_push: @@ -39,6 +38,20 @@ jobs: uses: docker/metadata-action@v5 with: images: ${{ env.IMAGE_NAME }} + + - name: Build Docker image + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64 + load: true + tags: ${{ env.TEST_TAG }} + build-args: | + JULIA_CPU_TARGET=generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1) + + - name: Test CLI + working-directory: ./test + run: source ./test-IFTPipeline.jl-cli.sh && IFT="docker run -v `pwd`:/app -w /app --rm ${{ env.TEST_TAG }}" preprocess_lopez input_data/ne-greenland.20220914.terra.250m - name: Build and push Docker image uses: docker/build-push-action@v6 @@ -47,4 +60,6 @@ jobs: platforms: linux/amd64 push: true tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file + labels: ${{ steps.meta.outputs.labels }} + build-args: | + JULIA_CPU_TARGET=generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1) \ No newline at end of file diff --git a/.github/workflows/build-docker-fetchdata.yml b/.github/workflows/build-docker-soit.yml similarity index 56% rename from .github/workflows/build-docker-fetchdata.yml rename to .github/workflows/build-docker-soit.yml index e6a0bdaa..6e356cb9 100644 --- a/.github/workflows/build-docker-fetchdata.yml +++ b/.github/workflows/build-docker-soit.yml @@ -1,6 +1,6 @@ # from https://docs.github.com/en/actions/guides/publishing-docker-images -name: Build and PushContainer - Fetchdata +name: Build and PushContainer - SOIT on: push: branches: @@ -13,7 +13,8 @@ on: env: REGISTRY: hub.docker.com - IMAGE_NAME: brownccv/icefloetracker-fetchdata + IMAGE_NAME: brownccv/icefloetracker-soit + TEST_TAG: brownccv/icefloetracker-soit:test jobs: build_and_push: @@ -39,11 +40,29 @@ jobs: with: images: ${{ env.IMAGE_NAME }} + - name: Build Docker image + uses: docker/build-push-action@v5 + with: + context: ./satellite-overpass-identification-tool + platforms: linux/amd64 + load: true + tags: ${{ env.TEST_TAG }} + + - name: Test SOIT + run: | + docker run -v `pwd`:/app -w /app --rm ${{ env.TEST_TAG }} \ + -u ${{ secrets.SPACEUSER }} -p ${{ secrets.SPACEPSWD }} \ + --startdate 2013-03-31 --enddate 2013-05-01 \ + --lat 76.0015 --lon -18.4315 \ + --csvoutpath . + + - name: Build and push Docker image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v5 with: - context: ./workflow/scripts + context: ./satellite-overpass-identification-tool platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + \ No newline at end of file diff --git a/.github/workflows/test-soit.yml b/.github/workflows/test-soit.yml index 8cb743b8..d9423b37 100644 --- a/.github/workflows/test-soit.yml +++ b/.github/workflows/test-soit.yml @@ -14,14 +14,18 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - - name: Install dependencies - run: pip install -r requirements.txt + - name: Run SOIT with a directory + run: | + pipx run --spec ./satellite-overpass-identification-tool soit \ + --SPACEUSER ${{ secrets.SPACEUSER }} --SPACEPSWD ${{ secrets.SPACEPSWD }} \ + --startdate 2013-03-31 --enddate 2013-05-01 \ + --lat 76.0015 --lon -18.4315 \ + --csvoutpath . - - name: Run SOIT + - name: Run SOIT with a particular path run: | - python ./workflow/scripts/pass_time_cylc.py -u ${{ secrets.SPACEUSER }} -p ${{ secrets.SPACEPSWD }} --startdate 2013-03-31 --enddate 2013-05-01 --lat 76.0015 --lon -18.4315 --csvoutpath . \ No newline at end of file + pipx run --spec ./satellite-overpass-identification-tool soit \ + --SPACEUSER ${{ secrets.SPACEUSER }} --SPACEPSWD ${{ secrets.SPACEPSWD }} \ + --startdate 2013-03-31 --enddate 2013-05-01 \ + --lat 76.0015 --lon -18.4315 \ + --csvoutpath tracker-results.csv diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5b33b1bf..e0e86d14 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,25 +25,21 @@ jobs: arch: x64 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: update pip - run: python -m pip install -U pip - - name: install python deps - run: python -m pip install -U -r requirements.txt - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - uses: julia-actions/cache@v2 + - run: julia --project=PythonSetupForIFTPipeline.jl PythonSetupForIFTPipeline.jl/setup.jl - uses: julia-actions/julia-buildpkg@v1 - env: - PYTHON: python + with: + project: IFTPipeline.jl - uses: julia-actions/julia-runtest@v1 - env: - PYTHON: python + with: + project: IFTPipeline.jl - uses: julia-actions/julia-processcoverage@v1 + with: + directories: IFTPipeline.jl/src - uses: codecov/codecov-action@v4 with: files: lcov.info diff --git a/.gitignore b/.gitignore index 6b409f0b..062614fb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ -Manifest.toml - resources/* !resources/README.md @@ -10,8 +8,6 @@ workflow/cylc-run/* workflow/report/* !workflow/report/README.md -test/__temp__ - icetest.sh trackplayground.jl @@ -20,6 +16,4 @@ trackplayground.jl *.simg -.JuliaFormatter.toml - -test/test_inputs/input_pipeline/*.jls \ No newline at end of file +runtime/ diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 00000000..04f91ba7 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,9 @@ +# Development Guide + +We have found VSCode to be a good IDE for development. You can use the [devcontainer](https://code.visualstudio.com/docs/devcontainers/containers) configuration to initialize a virtual machine with the Julia, Python and the geospatial dependencies for this project. + +Before running any Julia code for the first time, follow the instructions in +[PythonSetupForIFTPipeline.jl](./PythonSetupForIFTPipeline.jl/README.md) +to initialize a working Conda environment. + +Follow the instructions in [IFTPipeline.jl](./IFTPipeline.jl/) to run the tests of the pipeline. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a03d4ff9..afd5e516 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,28 @@ -FROM julia:1.9-bookworm -ENV TERM=xterm -ENV JULIA_PROJECT=/opt/ice-floe-tracker-pipeline -ENV JULIA_DEPOT_PATH=/opt/julia -ENV JULIA_PKGDIR=/opt/julia -ENV JULIA_BUILD='ENV["PYTHON"]=""; using Pkg; Pkg.build()' -ENV IFTPIPELINE_REPO='https://github.com/WilhelmusLab/ice-floe-tracker-pipeline.git' -ENV LOCAL_PATH_TO_IFT_CLI='/usr/local/bin/ice-floe-tracker.jl' +FROM --platform=$BUILDPLATFORM julia:1.11-bookworm +ARG TARGETARCH +ARG JULIA_CPU_TARGET="generic" +ENV JULIA_CPU_TARGET=${JULIA_CPU_TARGET} -WORKDIR /opt +# Dependencies +#=========================================== +ENV TERM=xterm -# DEPENDENCIES +# Python environment build #=========================================== -RUN apt-get -y update && \ - apt-get install -y git python3.10 && \ - rm -rf /var/lib/apt/list/* +ENV CONDA_JL_HOME=/opt/conda +ENV JULIA_DEPOT_PATH=/opt/julia +COPY ./PythonSetupForIFTPipeline.jl /opt/PythonSetupForIFTPipeline.jl +RUN julia --project="/opt/PythonSetupForIFTPipeline.jl" "/opt/PythonSetupForIFTPipeline.jl/setup.jl" -# Julia package build +# IFT Pipeline package build #=========================================== +COPY ./IFTPipeline.jl /opt/IFTPipeline.jl +RUN julia --project="/opt/IFTPipeline.jl" -e "using Pkg; Pkg.instantiate(); Pkg.precompile();" + +# Test the package +# RUN julia --project="/opt/IFTPipeline.jl" -e "using Pkg; Pkg.test();" -RUN git clone --single-branch --branch main --depth 1 ${IFTPIPELINE_REPO} -RUN /usr/local/julia/bin/julia --project=${JULIA_PROJECT} -e ${JULIA_BUILD} -RUN /usr/local/julia/bin/julia --project=${JULIA_PROJECT} -e 'using Pkg; Pkg.instantiate()' -COPY workflow/scripts/ice-floe-tracker.jl ${LOCAL_PATH_TO_IFT_CLI} -RUN chmod a+x ${LOCAL_PATH_TO_IFT_CLI} -ENV JULIA_DEPOT_PATH="/usr/local/bin/julia:$JULIA_DEPOT_PATH" -CMD [ "/bin/bash", "-c" ] +# CLI setup +#=========================================== +SHELL ["/bin/bash", "-c"] +ENTRYPOINT ["julia", "--project=/opt/IFTPipeline.jl", "/opt/IFTPipeline.jl/src/cli.jl" ] diff --git a/IFTPipeline.jl/Manifest.toml b/IFTPipeline.jl/Manifest.toml new file mode 100644 index 00000000..84987678 --- /dev/null +++ b/IFTPipeline.jl/Manifest.toml @@ -0,0 +1,1691 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.10.7" +manifest_format = "2.0" +project_hash = "ac968179f8dedb48fdd068dce54f394cecaece52" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.5.0" +weakdeps = ["ChainRulesCore", "Test"] + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + AbstractFFTsTestExt = "Test" + +[[deps.Accessors]] +deps = ["CompositionsBase", "ConstructionBase", "InverseFunctions", "LinearAlgebra", "MacroTools", "Markdown"] +git-tree-sha1 = "b392ede862e506d451fc1616e79aa6f4c673dab8" +uuid = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697" +version = "0.1.38" + + [deps.Accessors.extensions] + AccessorsAxisKeysExt = "AxisKeys" + AccessorsDatesExt = "Dates" + AccessorsIntervalSetsExt = "IntervalSets" + AccessorsStaticArraysExt = "StaticArrays" + AccessorsStructArraysExt = "StructArrays" + AccessorsTestExt = "Test" + AccessorsUnitfulExt = "Unitful" + + [deps.Accessors.weakdeps] + AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" + Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + Requires = "ae029012-a4dd-5104-9daa-d747884805df" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" + +[[deps.Adapt]] +deps = ["LinearAlgebra", "Requires"] +git-tree-sha1 = "50c3c56a52972d78e8be9fd135bfb91c9574c140" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "4.1.1" +weakdeps = ["StaticArrays"] + + [deps.Adapt.extensions] + AdaptStaticArraysExt = "StaticArrays" + +[[deps.ArgCheck]] +git-tree-sha1 = "a3a402a35a2f7e0b87828ccabbd5ebfbebe356b4" +uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" +version = "2.3.0" + +[[deps.ArgParse]] +deps = ["Logging", "TextWrap"] +git-tree-sha1 = "22cf435ac22956a7b45b0168abbc871176e7eecc" +uuid = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" +version = "1.2.0" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.ArnoldiMethod]] +deps = ["LinearAlgebra", "Random", "StaticArrays"] +git-tree-sha1 = "d57bd3762d308bded22c3b82d033bff85f6195c6" +uuid = "ec485272-7323-5ecc-a04f-4719b315124d" +version = "0.4.0" + +[[deps.ArrayInterface]] +deps = ["Adapt", "LinearAlgebra"] +git-tree-sha1 = "d5140b60b87473df18cf4fe66382b7c3596df047" +uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" +version = "7.17.1" + + [deps.ArrayInterface.extensions] + ArrayInterfaceBandedMatricesExt = "BandedMatrices" + ArrayInterfaceBlockBandedMatricesExt = "BlockBandedMatrices" + ArrayInterfaceCUDAExt = "CUDA" + ArrayInterfaceCUDSSExt = "CUDSS" + ArrayInterfaceChainRulesCoreExt = "ChainRulesCore" + ArrayInterfaceChainRulesExt = "ChainRules" + ArrayInterfaceGPUArraysCoreExt = "GPUArraysCore" + ArrayInterfaceReverseDiffExt = "ReverseDiff" + ArrayInterfaceSparseArraysExt = "SparseArrays" + ArrayInterfaceStaticArraysCoreExt = "StaticArraysCore" + ArrayInterfaceTrackerExt = "Tracker" + + [deps.ArrayInterface.weakdeps] + BandedMatrices = "aae01518-5342-5314-be14-df237901396f" + BlockBandedMatrices = "ffab5731-97b5-5995-9138-79e8c1846df0" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + CUDSS = "45b445bb-4962-46a0-9369-b4df9d0f772e" + ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2" + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + GPUArraysCore = "46192b85-c4d5-4398-a991-12ede77f4527" + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.AxisAlgorithms]] +deps = ["LinearAlgebra", "Random", "SparseArrays", "WoodburyMatrices"] +git-tree-sha1 = "66771c8d21c8ff5e3a93379480a2307ac36863f7" +uuid = "13072b0f-2c55-5437-9ae7-d433b7a33950" +version = "1.0.1" + +[[deps.AxisArrays]] +deps = ["Dates", "IntervalSets", "IterTools", "RangeArrays"] +git-tree-sha1 = "16351be62963a67ac4083f748fdb3cca58bfd52f" +uuid = "39de3d68-74b9-583c-8d2d-e117c070f3a9" +version = "0.4.7" + +[[deps.BangBang]] +deps = ["Accessors", "ConstructionBase", "InitialValues", "LinearAlgebra", "Requires"] +git-tree-sha1 = "e2144b631226d9eeab2d746ca8880b7ccff504ae" +uuid = "198e06fe-97b7-11e9-32a5-e1d131e6ad66" +version = "0.4.3" + + [deps.BangBang.extensions] + BangBangChainRulesCoreExt = "ChainRulesCore" + BangBangDataFramesExt = "DataFrames" + BangBangStaticArraysExt = "StaticArrays" + BangBangStructArraysExt = "StructArrays" + BangBangTablesExt = "Tables" + BangBangTypedTablesExt = "TypedTables" + + [deps.BangBang.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" + Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.Baselet]] +git-tree-sha1 = "aebf55e6d7795e02ca500a689d326ac979aaf89e" +uuid = "9718e550-a3fa-408a-8086-8db961cd8217" +version = "0.1.1" + +[[deps.BitFlags]] +git-tree-sha1 = "0691e34b3bb8be9307330f88d1a3c3f25466c24d" +uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" +version = "0.1.9" + +[[deps.BitTwiddlingConvenienceFunctions]] +deps = ["Static"] +git-tree-sha1 = "f21cfd4950cb9f0587d5067e69405ad2acd27b87" +uuid = "62783981-4cbd-42fc-bca8-16325de8dc4b" +version = "0.1.6" + +[[deps.CEnum]] +git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.5.0" + +[[deps.CPUSummary]] +deps = ["CpuId", "IfElse", "PrecompileTools", "Static"] +git-tree-sha1 = "5a97e67919535d6841172016c9530fd69494e5ec" +uuid = "2a0fbf3d-bb9c-48f3-b0a9-814d99fd7ab9" +version = "0.2.6" + +[[deps.CSV]] +deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] +git-tree-sha1 = "deddd8725e5e1cc49ee205a1964256043720a6c3" +uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +version = "0.10.15" + +[[deps.CSVFiles]] +deps = ["CodecZlib", "DataValues", "FileIO", "HTTP", "IterableTables", "IteratorInterfaceExtensions", "TableShowUtils", "TableTraits", "TableTraitsUtils", "TextParse"] +git-tree-sha1 = "2057fc0f258c2d67b31601d1a03319482a6f8b4c" +uuid = "5d742f6a-9f54-50ce-8119-2520741973ca" +version = "1.0.2" + +[[deps.CatIndices]] +deps = ["CustomUnitRanges", "OffsetArrays"] +git-tree-sha1 = "a0f80a09780eed9b1d106a1bf62041c2efc995bc" +uuid = "aafaddc9-749c-510e-ac4f-586e18779b91" +version = "0.2.2" + +[[deps.ChainRulesCore]] +deps = ["Compat", "LinearAlgebra"] +git-tree-sha1 = "3e4b134270b372f2ed4d4d0e936aabaefc1802bc" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "1.25.0" +weakdeps = ["SparseArrays"] + + [deps.ChainRulesCore.extensions] + ChainRulesCoreSparseArraysExt = "SparseArrays" + +[[deps.CloseOpenIntervals]] +deps = ["Static", "StaticArrayInterface"] +git-tree-sha1 = "05ba0d07cd4fd8b7a39541e31a7b0254704ea581" +uuid = "fb6a15b2-703c-40df-9091-08a04967cfa9" +version = "0.1.13" + +[[deps.Clustering]] +deps = ["Distances", "LinearAlgebra", "NearestNeighbors", "Printf", "Random", "SparseArrays", "Statistics", "StatsBase"] +git-tree-sha1 = "9ebb045901e9bbf58767a9f34ff89831ed711aae" +uuid = "aaaa29a8-35af-508c-8bc3-b662a17a0fe5" +version = "0.15.7" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "bce6804e5e6044c6daab27bb533d1295e4a2e759" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.6" + +[[deps.ColorSchemes]] +deps = ["ColorTypes", "ColorVectorSpace", "Colors", "FixedPointNumbers", "PrecompileTools", "Random"] +git-tree-sha1 = "b5278586822443594ff615963b0c09755771b3e0" +uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" +version = "3.26.0" + +[[deps.ColorTypes]] +deps = ["FixedPointNumbers", "Random"] +git-tree-sha1 = "b10d0b65641d57b8b4d5e234446582de5047050d" +uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +version = "0.11.5" + +[[deps.ColorVectorSpace]] +deps = ["ColorTypes", "FixedPointNumbers", "LinearAlgebra", "SpecialFunctions", "Statistics", "TensorCore"] +git-tree-sha1 = "600cc5508d66b78aae350f7accdb58763ac18589" +uuid = "c3611d14-8923-5661-9e6a-0046d554d3a4" +version = "0.9.10" + +[[deps.Colors]] +deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] +git-tree-sha1 = "362a287c3aa50601b0bc359053d5c2468f0e7ce0" +uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" +version = "0.12.11" + +[[deps.CommonWorldInvalidations]] +git-tree-sha1 = "ae52d1c52048455e85a387fbee9be553ec2b68d0" +uuid = "f70d9fcc-98c5-4d4a-abd7-e4cdeebd8ca8" +version = "1.0.0" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.16.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.1.1+0" + +[[deps.CompositionsBase]] +git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" +uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b" +version = "0.1.2" +weakdeps = ["InverseFunctions"] + + [deps.CompositionsBase.extensions] + CompositionsBaseInverseFunctionsExt = "InverseFunctions" + +[[deps.ComputationalResources]] +git-tree-sha1 = "52cb3ec90e8a8bea0e62e275ba577ad0f74821f7" +uuid = "ed09eef8-17a6-5b46-8889-db040fac31e3" +version = "0.3.2" + +[[deps.ConcurrentUtilities]] +deps = ["Serialization", "Sockets"] +git-tree-sha1 = "ea32b83ca4fefa1768dc84e504cc0a94fb1ab8d1" +uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" +version = "2.4.2" + +[[deps.Conda]] +deps = ["Downloads", "JSON", "VersionParsing"] +git-tree-sha1 = "b19db3927f0db4151cb86d073689f2428e524576" +uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d" +version = "1.10.2" + +[[deps.ConstructionBase]] +git-tree-sha1 = "76219f1ed5771adbb096743bff43fb5fdd4c1157" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.5.8" +weakdeps = ["IntervalSets", "LinearAlgebra", "StaticArrays"] + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseLinearAlgebraExt = "LinearAlgebra" + ConstructionBaseStaticArraysExt = "StaticArrays" + +[[deps.CoordinateTransformations]] +deps = ["LinearAlgebra", "StaticArrays"] +git-tree-sha1 = "f9d7112bfff8a19a3a4ea4e03a8e6a91fe8456bf" +uuid = "150eb455-5306-5404-9cee-2592286d6298" +version = "0.6.3" + +[[deps.CpuId]] +deps = ["Markdown"] +git-tree-sha1 = "fcbb72b032692610bfbdb15018ac16a36cf2e406" +uuid = "adafc99b-e345-5852-983c-f28acb93d879" +version = "0.3.1" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.CustomUnitRanges]] +git-tree-sha1 = "1a3f97f907e6dd8983b744d2642651bb162a3f7a" +uuid = "dc8bdbbb-1ca9-579f-8c36-e416f6a65cce" +version = "1.0.2" + +[[deps.DSP]] +deps = ["Compat", "FFTW", "IterTools", "LinearAlgebra", "Polynomials", "Random", "Reexport", "SpecialFunctions", "Statistics"] +git-tree-sha1 = "0df00546373af8eee1598fb4b2ba480b1ebe895c" +uuid = "717857b8-e6f2-59f4-9121-6e50c889abd2" +version = "0.7.10" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + +[[deps.DataFrames]] +deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "fb61b4812c49343d7ef0b533ba982c46021938a6" +uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +version = "1.7.0" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.20" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.DataValues]] +deps = ["DataValueInterfaces", "Dates"] +git-tree-sha1 = "d88a19299eba280a6d062e135a43f00323ae70bf" +uuid = "e7dc6d0d-1eca-5fa6-8ad6-5aecde8b7ea5" +version = "0.4.13" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.DefineSingletons]] +git-tree-sha1 = "0fba8b706d0178b4dc7fd44a96a92382c9065c2c" +uuid = "244e2a9f-e319-4986-a169-4d1fe445cd52" +version = "0.1.2" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" + +[[deps.Distances]] +deps = ["LinearAlgebra", "Statistics", "StatsAPI"] +git-tree-sha1 = "c7e3a542b999843086e2f29dac96a618c105be1d" +uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +version = "0.10.12" +weakdeps = ["ChainRulesCore", "SparseArrays"] + + [deps.Distances.extensions] + DistancesChainRulesCoreExt = "ChainRulesCore" + DistancesSparseArraysExt = "SparseArrays" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.3" + +[[deps.DoubleFloats]] +deps = ["GenericLinearAlgebra", "LinearAlgebra", "Polynomials", "Printf", "Quadmath", "Random", "Requires", "SpecialFunctions"] +git-tree-sha1 = "91482b72054f5feadb728cc382141a31e88dad21" +uuid = "497a8b3b-efae-58df-a0af-a86822472b78" +version = "1.4.2" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.ExceptionUnwrapping]] +deps = ["Test"] +git-tree-sha1 = "dcb08a0d93ec0b1cdc4af184b26b591e9695423a" +uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" +version = "0.1.10" + +[[deps.Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "cc5231d52eb1771251fbd37171dbc408bcc8a1b6" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.6.4+0" + +[[deps.ExternalDocstrings]] +git-tree-sha1 = "1224740fc4d07c989949e1c1b508ebd49a65a5f6" +uuid = "e189563c-0753-4f5e-ad5c-be4293c83fb4" +version = "0.1.1" + +[[deps.FFTViews]] +deps = ["CustomUnitRanges", "FFTW"] +git-tree-sha1 = "cbdf14d1e8c7c8aacbe8b19862e0179fd08321c2" +uuid = "4f61f5a4-77b1-5117-aa51-3ab5ef4ef0cd" +version = "0.3.2" + +[[deps.FFTW]] +deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] +git-tree-sha1 = "4820348781ae578893311153d69049a93d05f39d" +uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" +version = "1.8.0" + +[[deps.FFTW_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4d81ed14783ec49ce9f2e168208a12ce1815aa25" +uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" +version = "3.3.10+1" + +[[deps.FileIO]] +deps = ["Pkg", "Requires", "UUIDs"] +git-tree-sha1 = "62ca0547a14c57e98154423419d8a342dca75ca9" +uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +version = "1.16.4" + +[[deps.FilePathsBase]] +deps = ["Compat", "Dates"] +git-tree-sha1 = "7878ff7172a8e6beedd1dea14bd27c3c6340d361" +uuid = "48062228-2e41-5def-b9a4-89aafe57970f" +version = "0.9.22" +weakdeps = ["Mmap", "Test"] + + [deps.FilePathsBase.extensions] + FilePathsBaseMmapExt = "Mmap" + FilePathsBaseTestExt = "Test" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.FixedPointNumbers]] +deps = ["Statistics"] +git-tree-sha1 = "05882d6995ae5c12bb5f36dd2ed3f61c98cbb172" +uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +version = "0.8.5" + +[[deps.Folds]] +deps = ["Accessors", "BangBang", "Baselet", "DefineSingletons", "Distributed", "ExternalDocstrings", "InitialValues", "MicroCollections", "Referenceables", "Requires", "Test", "ThreadedScans", "Transducers"] +git-tree-sha1 = "7eb4bc88d8295e387a667fd43d67c157ddee76cf" +uuid = "41a02a25-b8f0-4f67-bc48-60067656b558" +version = "0.2.10" + + [deps.Folds.extensions] + FoldsOnlineStatsBaseExt = "OnlineStatsBase" + + [deps.Folds.weakdeps] + OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" + +[[deps.GenericLinearAlgebra]] +deps = ["LinearAlgebra", "Printf", "Random", "libblastrampoline_jll"] +git-tree-sha1 = "c4f9c87b74aedf20920034bd4db81d0bffc527d2" +uuid = "14197337-ba66-59df-a3e3-ca00e7dcff7a" +version = "0.3.14" + +[[deps.Ghostscript_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "43ba3d3c82c18d88471cfd2924931658838c9d8f" +uuid = "61579ee1-b43e-5ca0-a5da-69d92c66a64b" +version = "9.55.0+4" + +[[deps.Git]] +deps = ["Git_jll"] +git-tree-sha1 = "04eff47b1354d702c3a85e8ab23d539bb7d5957e" +uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" +version = "1.3.1" + +[[deps.Git_jll]] +deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] +git-tree-sha1 = "ea372033d09e4552a04fd38361cd019f9003f4f4" +uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" +version = "2.46.2+0" + +[[deps.Graphics]] +deps = ["Colors", "LinearAlgebra", "NaNMath"] +git-tree-sha1 = "a641238db938fff9b2f60d08ed9030387daf428c" +uuid = "a2bd30eb-e257-5431-a919-1863eab51364" +version = "1.1.3" + +[[deps.Graphs]] +deps = ["ArnoldiMethod", "Compat", "DataStructures", "Distributed", "Inflate", "LinearAlgebra", "Random", "SharedArrays", "SimpleTraits", "SparseArrays", "Statistics"] +git-tree-sha1 = "1dc470db8b1131cfc7fb4c115de89fe391b9e780" +uuid = "86223c79-3864-5bf0-83f7-82e725a168b6" +version = "1.12.0" + +[[deps.HDF5]] +deps = ["Compat", "HDF5_jll", "Libdl", "MPIPreferences", "Mmap", "Preferences", "Printf", "Random", "Requires", "UUIDs"] +git-tree-sha1 = "e856eef26cf5bf2b0f95f8f4fc37553c72c8641c" +uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" +version = "0.17.2" + + [deps.HDF5.extensions] + MPIExt = "MPI" + + [deps.HDF5.weakdeps] + MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" + +[[deps.HDF5_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"] +git-tree-sha1 = "38c8874692d48d5440d5752d6c74b0c6b0b60739" +uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" +version = "1.14.2+1" + +[[deps.HTTP]] +deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] +git-tree-sha1 = "1336e07ba2eb75614c99496501a8f4b233e9fafe" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "1.10.10" + +[[deps.HistogramThresholding]] +deps = ["ImageBase", "LinearAlgebra", "MappedArrays"] +git-tree-sha1 = "7194dfbb2f8d945abdaf68fa9480a965d6661e69" +uuid = "2c695a8d-9458-5d45-9878-1b8a99cf7853" +version = "0.3.1" + +[[deps.HostCPUFeatures]] +deps = ["BitTwiddlingConvenienceFunctions", "IfElse", "Libdl", "Static"] +git-tree-sha1 = "8e070b599339d622e9a081d17230d74a5c473293" +uuid = "3e5b6fbb-0976-4d2c-9146-d79de83f2fb0" +version = "0.1.17" + +[[deps.Hwloc_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "50aedf345a709ab75872f80a2779568dc0bb461b" +uuid = "e33a78d0-f292-5ffc-b300-72abe9b543c8" +version = "2.11.2+1" + +[[deps.IceFloeTracker]] +deps = ["AxisArrays", "Clustering", "ColorTypes", "CoordinateTransformations", "DSP", "DataFrames", "DataStructures", "Dates", "DelimitedFiles", "Distributed", "FFTW", "FileIO", "Git", "HDF5", "ImageAxes", "ImageBinarization", "ImageContrastAdjustment", "ImageCore", "ImageFiltering", "ImageIO", "ImageMorphology", "ImageSegmentation", "ImageTransformations", "Images", "Interpolations", "JLD2", "LinearAlgebra", "LoopVectorization", "MappedArrays", "OffsetArrays", "PaddedViews", "Peaks", "Pkg", "PositiveFactorizations", "Printf", "ProgressMeter", "PyCall", "Random", "Reexport", "Requires", "Rotations", "Serialization", "SharedArrays", "StaticArrays", "Statistics", "StatsBase", "TOML", "TiledIteration", "UpdateJulia"] +git-tree-sha1 = "ec0d100e3b10bd6009963544f8166f1ce903120d" +repo-rev = "main" +repo-url = "https://github.com/WilhelmusLab/IceFloeTracker.jl" +uuid = "04643c7a-9ac6-48c5-822f-2704f9e70bd3" +version = "0.6.0" + +[[deps.IfElse]] +git-tree-sha1 = "debdd00ffef04665ccbb3e150747a77560e8fad1" +uuid = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173" +version = "0.1.1" + +[[deps.ImageAxes]] +deps = ["AxisArrays", "ImageBase", "ImageCore", "Reexport", "SimpleTraits"] +git-tree-sha1 = "2e4520d67b0cef90865b3ef727594d2a58e0e1f8" +uuid = "2803e5a7-5153-5ecf-9a86-9b4c37f5f5ac" +version = "0.6.11" + +[[deps.ImageBase]] +deps = ["ImageCore", "Reexport"] +git-tree-sha1 = "b51bb8cae22c66d0f6357e3bcb6363145ef20835" +uuid = "c817782e-172a-44cc-b673-b171935fbb9e" +version = "0.1.5" + +[[deps.ImageBinarization]] +deps = ["HistogramThresholding", "ImageCore", "LinearAlgebra", "Polynomials", "Reexport", "Statistics"] +git-tree-sha1 = "33485b4e40d1df46c806498c73ea32dc17475c59" +uuid = "cbc4b850-ae4b-5111-9e64-df94c024a13d" +version = "0.3.1" + +[[deps.ImageContrastAdjustment]] +deps = ["ImageBase", "ImageCore", "ImageTransformations", "Parameters"] +git-tree-sha1 = "eb3d4365a10e3f3ecb3b115e9d12db131d28a386" +uuid = "f332f351-ec65-5f6a-b3d1-319c6670881a" +version = "0.3.12" + +[[deps.ImageCore]] +deps = ["AbstractFFTs", "ColorVectorSpace", "Colors", "FixedPointNumbers", "Graphics", "MappedArrays", "MosaicViews", "OffsetArrays", "PaddedViews", "Reexport"] +git-tree-sha1 = "acf614720ef026d38400b3817614c45882d75500" +uuid = "a09fc81d-aa75-5fe9-8630-4744c3626534" +version = "0.9.4" + +[[deps.ImageDistances]] +deps = ["Distances", "ImageCore", "ImageMorphology", "LinearAlgebra", "Statistics"] +git-tree-sha1 = "08b0e6354b21ef5dd5e49026028e41831401aca8" +uuid = "51556ac3-7006-55f5-8cb3-34580c88182d" +version = "0.2.17" + +[[deps.ImageFiltering]] +deps = ["CatIndices", "ComputationalResources", "DataStructures", "FFTViews", "FFTW", "ImageBase", "ImageCore", "LinearAlgebra", "OffsetArrays", "PrecompileTools", "Reexport", "SparseArrays", "StaticArrays", "Statistics", "TiledIteration"] +git-tree-sha1 = "3447781d4c80dbe6d71d239f7cfb1f8049d4c84f" +uuid = "6a3955dd-da59-5b1f-98d4-e7296123deb5" +version = "0.7.6" + +[[deps.ImageIO]] +deps = ["FileIO", "IndirectArrays", "JpegTurbo", "LazyModules", "Netpbm", "OpenEXR", "PNGFiles", "QOI", "Sixel", "TiffImages", "UUIDs"] +git-tree-sha1 = "437abb322a41d527c197fa800455f79d414f0a3c" +uuid = "82e4d734-157c-48bb-816b-45c225c6df19" +version = "0.6.8" + +[[deps.ImageMagick]] +deps = ["FileIO", "ImageCore", "ImageMagick_jll", "InteractiveUtils", "Libdl", "Pkg", "Random"] +git-tree-sha1 = "5bc1cb62e0c5f1005868358db0692c994c3a13c6" +uuid = "6218d12a-5da1-5696-b52f-db25d2ecc6d1" +version = "1.2.1" + +[[deps.ImageMagick_jll]] +deps = ["Artifacts", "Ghostscript_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "OpenJpeg_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "d65554bad8b16d9562050c67e7223abf91eaba2f" +uuid = "c73af94c-d91f-53ed-93a7-00f77d67a9d7" +version = "6.9.13+0" + +[[deps.ImageMetadata]] +deps = ["AxisArrays", "ImageAxes", "ImageBase", "ImageCore"] +git-tree-sha1 = "355e2b974f2e3212a75dfb60519de21361ad3cb7" +uuid = "bc367c6b-8a6b-528e-b4bd-a4b897500b49" +version = "0.9.9" + +[[deps.ImageMorphology]] +deps = ["ImageCore", "LinearAlgebra", "Requires", "TiledIteration"] +git-tree-sha1 = "e7c68ab3df4a75511ba33fc5d8d9098007b579a8" +uuid = "787d08f9-d448-5407-9aad-5290dd7ab264" +version = "0.3.2" + +[[deps.ImageQualityIndexes]] +deps = ["ImageContrastAdjustment", "ImageCore", "ImageDistances", "ImageFiltering", "LazyModules", "OffsetArrays", "PrecompileTools", "Statistics"] +git-tree-sha1 = "783b70725ed326340adf225be4889906c96b8fd1" +uuid = "2996bd0c-7a13-11e9-2da2-2f5ce47296a9" +version = "0.3.7" + +[[deps.ImageSegmentation]] +deps = ["Clustering", "DataStructures", "Distances", "Graphs", "ImageCore", "ImageFiltering", "ImageMorphology", "LinearAlgebra", "MetaGraphs", "RegionTrees", "SimpleWeightedGraphs", "StaticArrays", "Statistics"] +git-tree-sha1 = "44664eea5408828c03e5addb84fa4f916132fc26" +uuid = "80713f31-8817-5129-9cf8-209ff8fb23e1" +version = "1.8.1" + +[[deps.ImageShow]] +deps = ["Base64", "ColorSchemes", "FileIO", "ImageBase", "ImageCore", "OffsetArrays", "StackViews"] +git-tree-sha1 = "3b5344bcdbdc11ad58f3b1956709b5b9345355de" +uuid = "4e3cecfd-b093-5904-9786-8bbb286a6a31" +version = "0.3.8" + +[[deps.ImageTransformations]] +deps = ["AxisAlgorithms", "ColorVectorSpace", "CoordinateTransformations", "ImageBase", "ImageCore", "Interpolations", "OffsetArrays", "Rotations", "StaticArrays"] +git-tree-sha1 = "8717482f4a2108c9358e5c3ca903d3a6113badc9" +uuid = "02fcd773-0e25-5acc-982a-7f6622650795" +version = "0.9.5" + +[[deps.Images]] +deps = ["Base64", "FileIO", "Graphics", "ImageAxes", "ImageBase", "ImageContrastAdjustment", "ImageCore", "ImageDistances", "ImageFiltering", "ImageIO", "ImageMagick", "ImageMetadata", "ImageMorphology", "ImageQualityIndexes", "ImageSegmentation", "ImageShow", "ImageTransformations", "IndirectArrays", "IntegralArrays", "Random", "Reexport", "SparseArrays", "StaticArrays", "Statistics", "StatsBase", "TiledIteration"] +git-tree-sha1 = "5fa9f92e1e2918d9d1243b1131abe623cdf98be7" +uuid = "916415d5-f1e6-5110-898d-aaa5f9f070e0" +version = "0.25.3" + +[[deps.Imath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "0936ba688c6d201805a83da835b55c61a180db52" +uuid = "905a6f67-0a94-5f89-b386-d35d92009cd1" +version = "3.1.11+0" + +[[deps.IndirectArrays]] +git-tree-sha1 = "012e604e1c7458645cb8b436f8fba789a51b257f" +uuid = "9b13fd28-a010-5f03-acff-a1bbcff69959" +version = "1.0.0" + +[[deps.Inflate]] +git-tree-sha1 = "d1b1b796e47d94588b3757fe84fbf65a5ec4a80d" +uuid = "d25df0c9-e2be-5dd7-82c8-3ad0b3e990b9" +version = "0.1.5" + +[[deps.InitialValues]] +git-tree-sha1 = "4da0f88e9a39111c2fa3add390ab15f3a44f3ca3" +uuid = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" +version = "0.3.1" + +[[deps.InlineStrings]] +git-tree-sha1 = "45521d31238e87ee9f9732561bfee12d4eebd52d" +uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" +version = "1.4.2" + + [deps.InlineStrings.extensions] + ArrowTypesExt = "ArrowTypes" + ParsersExt = "Parsers" + + [deps.InlineStrings.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" + +[[deps.IntegralArrays]] +deps = ["ColorTypes", "FixedPointNumbers", "IntervalSets"] +git-tree-sha1 = "b842cbff3f44804a84fda409745cc8f04c029a20" +uuid = "1d092043-8f09-5a30-832f-7509e371ab51" +version = "0.1.6" + +[[deps.IntelOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] +git-tree-sha1 = "10bd689145d2c3b2a9844005d01087cc1194e79e" +uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" +version = "2024.2.1+0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.Interpolations]] +deps = ["AxisAlgorithms", "ChainRulesCore", "LinearAlgebra", "OffsetArrays", "Random", "Ratios", "Requires", "SharedArrays", "SparseArrays", "StaticArrays", "WoodburyMatrices"] +git-tree-sha1 = "00a19d6ab0cbdea2978fc23c5a6482e02c192501" +uuid = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" +version = "0.14.0" + +[[deps.IntervalSets]] +git-tree-sha1 = "dba9ddf07f77f60450fe5d2e2beb9854d9a49bd0" +uuid = "8197267c-284f-5f27-9208-e0e47529a953" +version = "0.7.10" +weakdeps = ["Random", "RecipesBase", "Statistics"] + + [deps.IntervalSets.extensions] + IntervalSetsRandomExt = "Random" + IntervalSetsRecipesBaseExt = "RecipesBase" + IntervalSetsStatisticsExt = "Statistics" + +[[deps.InverseFunctions]] +git-tree-sha1 = "a779299d77cd080bf77b97535acecd73e1c5e5cb" +uuid = "3587e190-3f89-42d0-90ee-14403ec27112" +version = "0.1.17" +weakdeps = ["Dates", "Test"] + + [deps.InverseFunctions.extensions] + InverseFunctionsDatesExt = "Dates" + InverseFunctionsTestExt = "Test" + +[[deps.InvertedIndices]] +git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" +uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +version = "1.3.0" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.2" + +[[deps.IterTools]] +git-tree-sha1 = "42d5f897009e7ff2cf88db414a389e5ed1bdd023" +uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" +version = "1.10.0" + +[[deps.IterableTables]] +deps = ["DataValues", "IteratorInterfaceExtensions", "Requires", "TableTraits", "TableTraitsUtils"] +git-tree-sha1 = "70300b876b2cebde43ebc0df42bc8c94a144e1b4" +uuid = "1c8ee90f-4401-5389-894e-7a04a3dc0f4d" +version = "1.0.0" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLD2]] +deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"] +git-tree-sha1 = "a0746c21bdc986d0dc293efa6b1faee112c37c28" +uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +version = "0.4.53" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "be3dc50a92e5a386872a493a10050136d4703f9b" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.6.1" + +[[deps.JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.4" + +[[deps.JpegTurbo]] +deps = ["CEnum", "FileIO", "ImageCore", "JpegTurbo_jll", "TOML"] +git-tree-sha1 = "fa6d0bcff8583bac20f1ffa708c3913ca605c611" +uuid = "b835a17e-a41a-41e7-81f0-2f016b05efe0" +version = "0.1.5" + +[[deps.JpegTurbo_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "25ee0be4d43d0269027024d75a24c24d6c6e590c" +uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8" +version = "3.0.4+0" + +[[deps.LERC_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "36bdbc52f13a7d1dcb0f3cd694e01677a515655b" +uuid = "88015f11-f218-50d7-93a8-a6af411a945d" +version = "4.0.0+0" + +[[deps.LLVMOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "78211fb6cbc872f77cad3fc0b6cf647d923f4929" +uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" +version = "18.1.7+0" + +[[deps.LaTeXStrings]] +git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.4.0" + +[[deps.LayoutPointers]] +deps = ["ArrayInterface", "LinearAlgebra", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface"] +git-tree-sha1 = "a9eaadb366f5493a5654e843864c13d8b107548c" +uuid = "10f19ff3-798f-405d-979b-55457f8fc047" +version = "0.1.17" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" + +[[deps.LazyModules]] +git-tree-sha1 = "a560dd966b386ac9ae60bdd3a3d3a326062d3c3e" +uuid = "8cdb02fc-e678-4876-92c5-9defec4f444e" +version = "0.3.1" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.4.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.6.4+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "61dfdba58e585066d8bce214c5a51eaa0539f269" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.17.0+1" + +[[deps.Libtiff_jll]] +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "LERC_jll", "Libdl", "XZ_jll", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "b404131d06f7886402758c9ce2214b636eb4d54a" +uuid = "89763e89-9b03-5906-acba-b20f662cd828" +version = "4.7.0+0" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.LittleCMS_jll]] +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll"] +git-tree-sha1 = "fa7fd067dca76cadd880f1ca937b4f387975a9f5" +uuid = "d3a379c0-f9a3-5b72-a4c0-6bf4d2e8af0f" +version = "2.16.0+0" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "a2d09619db4e765091ee5c6ffe8872849de0feea" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.28" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.LoggingExtras]] +deps = ["Dates", "Logging"] +git-tree-sha1 = "f02b56007b064fbfddb4c9cd60161b6dd0f40df3" +uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" +version = "1.1.0" + +[[deps.LoopVectorization]] +deps = ["ArrayInterface", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "PrecompileTools", "SIMDTypes", "SLEEFPirates", "Static", "StaticArrayInterface", "ThreadingUtilities", "UnPack", "VectorizationBase"] +git-tree-sha1 = "8084c25a250e00ae427a379a5b607e7aed96a2dd" +uuid = "bdcacae8-1622-11e9-2a5c-532679323890" +version = "0.12.171" + + [deps.LoopVectorization.extensions] + ForwardDiffExt = ["ChainRulesCore", "ForwardDiff"] + SpecialFunctionsExt = "SpecialFunctions" + + [deps.LoopVectorization.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" + +[[deps.MKL_jll]] +deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"] +git-tree-sha1 = "f046ccd0c6db2832a9f639e2c669c6fe867e5f4f" +uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" +version = "2024.2.0+0" + +[[deps.MPICH_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] +git-tree-sha1 = "7715e65c47ba3941c502bffb7f266a41a7f54423" +uuid = "7cb0a576-ebde-5e09-9194-50597f1243b4" +version = "4.2.3+0" + +[[deps.MPIPreferences]] +deps = ["Libdl", "Preferences"] +git-tree-sha1 = "c105fe467859e7f6e9a852cb15cb4301126fac07" +uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" +version = "0.1.11" + +[[deps.MPItrampoline_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] +git-tree-sha1 = "70e830dab5d0775183c99fc75e4c24c614ed7142" +uuid = "f1f71cc9-e9ae-5b93-9b94-4fe0e1ad3748" +version = "5.5.1+0" + +[[deps.MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "2fa9ee3e63fd3a4f7a9a4f4744a52f4856de82df" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.13" + +[[deps.ManualMemory]] +git-tree-sha1 = "bcaef4fc7a0cfe2cba636d84cda54b5e4e4ca3cd" +uuid = "d125e4d3-2237-4719-b19c-fa641b8a4667" +version = "0.1.8" + +[[deps.MappedArrays]] +git-tree-sha1 = "2dab0221fe2b0f2cb6754eaa743cc266339f527e" +uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" +version = "0.4.2" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] +git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.1.9" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+1" + +[[deps.MetaGraphs]] +deps = ["Graphs", "JLD2", "Random"] +git-tree-sha1 = "1130dbe1d5276cb656f6e1094ce97466ed700e5a" +uuid = "626554b9-1ddb-594c-aa3c-2596fe9399a5" +version = "0.7.2" + +[[deps.MicroCollections]] +deps = ["Accessors", "BangBang", "InitialValues"] +git-tree-sha1 = "44d32db644e84c75dab479f1bc15ee76a1a3618f" +uuid = "128add7d-3638-4c79-886c-908ea0c25c34" +version = "0.2.0" + +[[deps.MicrosoftMPI_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f12a29c4400ba812841c6ace3f4efbb6dbb3ba01" +uuid = "9237b28f-5490-5468-be7b-bb81f5f5e6cf" +version = "10.1.4+2" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.2.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.MosaicViews]] +deps = ["MappedArrays", "OffsetArrays", "PaddedViews", "StackViews"] +git-tree-sha1 = "7b86a5d4d70a9f5cdf2dacb3cbe6d251d1a61dbe" +uuid = "e94cdb99-869f-56ef-bcf0-1ae2bcbe0389" +version = "0.3.4" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2023.1.10" + +[[deps.NaNMath]] +deps = ["OpenLibm_jll"] +git-tree-sha1 = "0877504529a3e5c3343c6f8b4c0381e57e4387e4" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "1.0.2" + +[[deps.NearestNeighbors]] +deps = ["Distances", "StaticArrays"] +git-tree-sha1 = "8a3271d8309285f4db73b4f662b1b290c715e85e" +uuid = "b8a86587-4115-5ab1-83bc-aa920d37bbce" +version = "0.4.21" + +[[deps.Netpbm]] +deps = ["FileIO", "ImageCore", "ImageMetadata"] +git-tree-sha1 = "d92b107dbb887293622df7697a2223f9f8176fcd" +uuid = "f09324ee-3d7c-5217-9330-fc30815ba969" +version = "1.1.1" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.Nullables]] +git-tree-sha1 = "8f87854cc8f3685a60689d8edecaa29d2251979b" +uuid = "4d1e1d77-625e-5b40-9113-a560ec7a8ecd" +version = "1.0.0" + +[[deps.OffsetArrays]] +git-tree-sha1 = "1a27764e945a152f7ca7efa04de513d473e9542e" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.14.1" +weakdeps = ["Adapt"] + + [deps.OffsetArrays.extensions] + OffsetArraysAdaptExt = "Adapt" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.23+4" + +[[deps.OpenEXR]] +deps = ["Colors", "FileIO", "OpenEXR_jll"] +git-tree-sha1 = "97db9e07fe2091882c765380ef58ec553074e9c7" +uuid = "52e1d378-f018-4a11-a4be-720524705ac7" +version = "0.3.3" + +[[deps.OpenEXR_jll]] +deps = ["Artifacts", "Imath_jll", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "8292dd5c8a38257111ada2174000a33745b06d4e" +uuid = "18a262bb-aa17-5467-a713-aee519bc75cb" +version = "3.2.4+0" + +[[deps.OpenJpeg_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libtiff_jll", "LittleCMS_jll", "libpng_jll"] +git-tree-sha1 = "f4cb457ffac5f5cf695699f82c537073958a6a6c" +uuid = "643b3616-a352-519d-856d-80112ee9badc" +version = "2.5.2+0" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.1+2" + +[[deps.OpenMPI_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML", "Zlib_jll"] +git-tree-sha1 = "bfce6d523861a6c562721b262c0d1aaeead2647f" +uuid = "fe0851c0-eecd-5654-98d4-656369965a5c" +version = "5.0.5+0" + +[[deps.OpenSSL]] +deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"] +git-tree-sha1 = "38cb508d080d21dc1128f7fb04f20387ed4c0af4" +uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" +version = "1.4.3" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "7493f61f55a6cce7325f197443aa80d32554ba10" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.0.15+1" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.5+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.3" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.42.0+1" + +[[deps.PNGFiles]] +deps = ["Base64", "CEnum", "ImageCore", "IndirectArrays", "OffsetArrays", "libpng_jll"] +git-tree-sha1 = "67186a2bc9a90f9f85ff3cc8277868961fb57cbd" +uuid = "f57f5aa1-a3ce-4bc8-8ab9-96f992907883" +version = "0.4.3" + +[[deps.PaddedViews]] +deps = ["OffsetArrays"] +git-tree-sha1 = "0fac6313486baae819364c52b4f483450a9d793f" +uuid = "5432bcbf-9aad-5242-b902-cca2824c8663" +version = "0.5.12" + +[[deps.Parameters]] +deps = ["OrderedCollections", "UnPack"] +git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe" +uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a" +version = "0.12.3" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.8.1" + +[[deps.Peaks]] +deps = ["RecipesBase", "SIMD"] +git-tree-sha1 = "75d0ce1c30696d77bc60840222d7fc5d549ebf5f" +uuid = "18e31ff7-3703-566c-8e60-38913d67486b" +version = "0.5.3" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.10.0" + +[[deps.PkgVersion]] +deps = ["Pkg"] +git-tree-sha1 = "f9501cc0430a26bc3d156ae1b5b0c1b47af4d6da" +uuid = "eebad327-c553-4316-9ea0-9fa01ccd7688" +version = "0.3.3" + +[[deps.PolyesterWeave]] +deps = ["BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "Static", "ThreadingUtilities"] +git-tree-sha1 = "645bed98cd47f72f67316fd42fc47dee771aefcd" +uuid = "1d0040c9-8b98-4ee7-8388-3f51789ca0ad" +version = "0.2.2" + +[[deps.Polynomials]] +deps = ["LinearAlgebra", "RecipesBase", "Requires", "Setfield", "SparseArrays"] +git-tree-sha1 = "1a9cfb2dc2c2f1bd63f1906d72af39a79b49b736" +uuid = "f27b6e38-b328-58d1-80ce-0feddd5e7a45" +version = "4.0.11" + + [deps.Polynomials.extensions] + PolynomialsChainRulesCoreExt = "ChainRulesCore" + PolynomialsFFTWExt = "FFTW" + PolynomialsMakieCoreExt = "MakieCore" + PolynomialsMutableArithmeticsExt = "MutableArithmetics" + + [deps.Polynomials.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" + MakieCore = "20f20a25-4f0e-4fdf-b5d1-57303727442b" + MutableArithmetics = "d8a4904e-b15c-11e9-3269-09a3773c0cb0" + +[[deps.PooledArrays]] +deps = ["DataAPI", "Future"] +git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" +uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" +version = "1.4.3" + +[[deps.PositiveFactorizations]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "17275485f373e6673f7e7f97051f703ed5b15b20" +uuid = "85a6dd25-e78a-55b7-8502-1745935b8125" +version = "0.2.4" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.1" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.3" + +[[deps.PrettyTables]] +deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "1101cd475833706e4d0e7b122218257178f48f34" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "2.4.0" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.ProgressMeter]] +deps = ["Distributed", "Printf"] +git-tree-sha1 = "8f6bc219586aef8baf0ff9a5fe16ee9c70cb65e4" +uuid = "92933f4c-e287-5a05-a399-4b506db050ca" +version = "1.10.2" + +[[deps.PyCall]] +deps = ["Conda", "Dates", "Libdl", "LinearAlgebra", "MacroTools", "Serialization", "VersionParsing"] +git-tree-sha1 = "9816a3826b0ebf49ab4926e2b18842ad8b5c8f04" +uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +version = "1.96.4" + +[[deps.QOI]] +deps = ["ColorTypes", "FileIO", "FixedPointNumbers"] +git-tree-sha1 = "8b3fc30bc0390abdce15f8822c889f669baed73d" +uuid = "4b34888f-f399-49d4-9bb3-47ed5cae4e65" +version = "1.0.1" + +[[deps.Quadmath]] +deps = ["Compat", "Printf", "Random", "Requires"] +git-tree-sha1 = "67fe599f02c3f7be5d97310674cd05429d6f1b42" +uuid = "be4d8f0f-7fa4-5f49-b795-2f01399ab2dd" +version = "0.5.10" + +[[deps.Quaternions]] +deps = ["LinearAlgebra", "Random", "RealDot"] +git-tree-sha1 = "994cc27cdacca10e68feb291673ec3a76aa2fae9" +uuid = "94ee1d12-ae83-5a48-8b1c-48b8ff168ae0" +version = "0.7.6" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.RangeArrays]] +git-tree-sha1 = "b9039e93773ddcfc828f12aadf7115b4b4d225f5" +uuid = "b3c3ace0-ae52-54e7-9d0b-2c1406fd6b9d" +version = "0.3.2" + +[[deps.Ratios]] +deps = ["Requires"] +git-tree-sha1 = "1342a47bf3260ee108163042310d26f2be5ec90b" +uuid = "c84ed2f1-dad5-54f0-aa8e-dbefe2724439" +version = "0.4.5" +weakdeps = ["FixedPointNumbers"] + + [deps.Ratios.extensions] + RatiosFixedPointNumbersExt = "FixedPointNumbers" + +[[deps.RealDot]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" +uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" +version = "0.1.0" + +[[deps.RecipesBase]] +deps = ["PrecompileTools"] +git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.3.4" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.Referenceables]] +deps = ["Adapt"] +git-tree-sha1 = "02d31ad62838181c1a3a5fd23a1ce5914a643601" +uuid = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" +version = "0.1.3" + +[[deps.RegionTrees]] +deps = ["IterTools", "LinearAlgebra", "StaticArrays"] +git-tree-sha1 = "4618ed0da7a251c7f92e869ae1a19c74a7d2a7f9" +uuid = "dee08c22-ab7f-5625-9660-a9af2021b33f" +version = "0.3.2" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + +[[deps.Rotations]] +deps = ["LinearAlgebra", "Quaternions", "Random", "StaticArrays"] +git-tree-sha1 = "5680a9276685d392c87407df00d57c9924d9f11e" +uuid = "6038ab10-8711-5258-84ad-4b1120ba62dc" +version = "1.7.1" +weakdeps = ["RecipesBase"] + + [deps.Rotations.extensions] + RotationsRecipesBaseExt = "RecipesBase" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.SIMD]] +deps = ["PrecompileTools"] +git-tree-sha1 = "52af86e35dd1b177d051b12681e1c581f53c281b" +uuid = "fdea26ae-647d-5447-a871-4b548cad5224" +version = "3.7.0" + +[[deps.SIMDTypes]] +git-tree-sha1 = "330289636fb8107c5f32088d2741e9fd7a061a5c" +uuid = "94e857df-77ce-4151-89e5-788b33177be4" +version = "0.1.0" + +[[deps.SLEEFPirates]] +deps = ["IfElse", "Static", "VectorizationBase"] +git-tree-sha1 = "456f610ca2fbd1c14f5fcf31c6bfadc55e7d66e0" +uuid = "476501e8-09a2-5ece-8869-fb82de89a1fa" +version = "0.6.43" + +[[deps.SentinelArrays]] +deps = ["Dates", "Random"] +git-tree-sha1 = "d0553ce4031a081cc42387a9b9c8441b7d99f32d" +uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" +version = "1.4.7" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.Setfield]] +deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"] +git-tree-sha1 = "e2cc6d8c88613c05e1defb55170bf5ff211fbeac" +uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" +version = "1.1.1" + +[[deps.SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[deps.SimpleBufferStream]] +git-tree-sha1 = "f305871d2f381d21527c770d4788c06c097c9bc1" +uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" +version = "1.2.0" + +[[deps.SimpleTraits]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231" +uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" +version = "0.9.4" + +[[deps.SimpleWeightedGraphs]] +deps = ["Graphs", "LinearAlgebra", "Markdown", "SparseArrays"] +git-tree-sha1 = "4b33e0e081a825dbfaf314decf58fa47e53d6acb" +uuid = "47aef6b3-ad0c-573a-a1e2-d07658019622" +version = "1.4.0" + +[[deps.Sixel]] +deps = ["Dates", "FileIO", "ImageCore", "IndirectArrays", "OffsetArrays", "REPL", "libsixel_jll"] +git-tree-sha1 = "2da10356e31327c7096832eb9cd86307a50b1eb6" +uuid = "45858cf5-a6b0-47a3-bbea-62219f50df47" +version = "0.1.3" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.2.1" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.10.0" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "2f5d4697f21388cbe1ff299430dd169ef97d7e14" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.4.0" +weakdeps = ["ChainRulesCore"] + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + +[[deps.SplittablesBase]] +deps = ["Setfield", "Test"] +git-tree-sha1 = "e08a62abc517eb79667d0a29dc08a3b589516bb5" +uuid = "171d559e-b47b-412a-8079-5efa626c420e" +version = "0.1.15" + +[[deps.StackViews]] +deps = ["OffsetArrays"] +git-tree-sha1 = "46e589465204cd0c08b4bd97385e4fa79a0c770c" +uuid = "cae243ae-269e-4f55-b966-ac2d0dc13c15" +version = "0.1.1" + +[[deps.Static]] +deps = ["CommonWorldInvalidations", "IfElse", "PrecompileTools"] +git-tree-sha1 = "87d51a3ee9a4b0d2fe054bdd3fc2436258db2603" +uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3" +version = "1.1.1" + +[[deps.StaticArrayInterface]] +deps = ["ArrayInterface", "Compat", "IfElse", "LinearAlgebra", "PrecompileTools", "Static"] +git-tree-sha1 = "96381d50f1ce85f2663584c8e886a6ca97e60554" +uuid = "0d7ed370-da01-4f52-bd93-41d350b8b718" +version = "1.8.0" +weakdeps = ["OffsetArrays", "StaticArrays"] + + [deps.StaticArrayInterface.extensions] + StaticArrayInterfaceOffsetArraysExt = "OffsetArrays" + StaticArrayInterfaceStaticArraysExt = "StaticArrays" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "777657803913ffc7e8cc20f0fd04b634f871af8f" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.9.8" +weakdeps = ["ChainRulesCore", "Statistics"] + + [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" + StaticArraysStatisticsExt = "Statistics" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "192954ef1208c7019899fbf8049e717f92959682" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.3" + +[[deps.Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.10.0" + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.7.0" + +[[deps.StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "5cf7606d6cef84b543b483848d4ae08ad9832b21" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.3" + +[[deps.StringManipulation]] +deps = ["PrecompileTools"] +git-tree-sha1 = "a6b1675a536c5ad1a60e5a5153e1fee12eb146e3" +uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" +version = "0.4.0" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.2.1+1" + +[[deps.Suppressor]] +deps = ["Logging"] +git-tree-sha1 = "6dbb5b635c5437c68c28c2ac9e39b87138f37c0a" +uuid = "fd094767-a336-5f1f-9728-57cf17d0bbfb" +version = "0.2.8" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableShowUtils]] +deps = ["DataValues", "Dates", "JSON", "Markdown", "Unicode"] +git-tree-sha1 = "2a41a3dedda21ed1184a47caab56ed9304e9a038" +uuid = "5e66a065-1f0a-5976-b372-e0b8c017ca10" +version = "0.2.6" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.TableTraitsUtils]] +deps = ["DataValues", "IteratorInterfaceExtensions", "Missings", "TableTraits"] +git-tree-sha1 = "78fecfe140d7abb480b53a44f3f85b6aa373c293" +uuid = "382cd787-c1b6-5bf2-a167-d5b971a19bda" +version = "1.0.2" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] +git-tree-sha1 = "598cd7c1f68d1e205689b1c2fe65a9f85846f297" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.12.0" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.TensorCore]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "1feb45f88d133a655e001435632f019a9a1bcdb6" +uuid = "62fd8b95-f654-4bbd-a8a5-9c27f68ccd50" +version = "0.1.1" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.TextParse]] +deps = ["CodecZlib", "DataStructures", "Dates", "DoubleFloats", "Mmap", "Nullables", "WeakRefStrings"] +git-tree-sha1 = "eb1f4fb185c8644faa2d18d14c72f2c24412415f" +uuid = "e0df1984-e451-5cb5-8b61-797a481e67e3" +version = "1.0.2" + +[[deps.TextWrap]] +git-tree-sha1 = "43044b737fa70bc12f6105061d3da38f881a3e3c" +uuid = "b718987f-49a8-5099-9789-dcd902bef87d" +version = "1.0.2" + +[[deps.ThreadedScans]] +deps = ["ArgCheck"] +git-tree-sha1 = "ca1ba3000289eacba571aaa4efcefb642e7a1de6" +uuid = "24d252fe-5d94-4a69-83ea-56a14333d47a" +version = "0.1.0" + +[[deps.ThreadingUtilities]] +deps = ["ManualMemory"] +git-tree-sha1 = "eda08f7e9818eb53661b3deb74e3159460dfbc27" +uuid = "8290d209-cae3-49c0-8002-c8c24d57dab5" +version = "0.5.2" + +[[deps.TiffImages]] +deps = ["ColorTypes", "DataStructures", "DocStringExtensions", "FileIO", "FixedPointNumbers", "IndirectArrays", "Inflate", "Mmap", "OffsetArrays", "PkgVersion", "ProgressMeter", "SIMD", "UUIDs"] +git-tree-sha1 = "38f139cc4abf345dd4f22286ec000728d5e8e097" +uuid = "731e570b-9d59-4bfa-96dc-6df516fadf69" +version = "0.10.2" + +[[deps.TiledIteration]] +deps = ["OffsetArrays"] +git-tree-sha1 = "5683455224ba92ef59db72d10690690f4a8dc297" +uuid = "06e1c1a7-607b-532d-9fad-de7d9aa2abac" +version = "0.3.1" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.3" + +[[deps.Transducers]] +deps = ["Accessors", "ArgCheck", "BangBang", "Baselet", "CompositionsBase", "ConstructionBase", "DefineSingletons", "Distributed", "InitialValues", "Logging", "Markdown", "MicroCollections", "Requires", "SplittablesBase", "Tables"] +git-tree-sha1 = "7deeab4ff96b85c5f72c824cae53a1398da3d1cb" +uuid = "28d57a85-8fef-5791-bfe6-a80928e7c999" +version = "0.4.84" + + [deps.Transducers.extensions] + TransducersAdaptExt = "Adapt" + TransducersBlockArraysExt = "BlockArrays" + TransducersDataFramesExt = "DataFrames" + TransducersLazyArraysExt = "LazyArrays" + TransducersOnlineStatsBaseExt = "OnlineStatsBase" + TransducersReferenceablesExt = "Referenceables" + + [deps.Transducers.weakdeps] + Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" + BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" + OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" + Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" + +[[deps.URIs]] +git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.5.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.UnPack]] +git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b" +uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" +version = "1.0.2" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.UpdateJulia]] +deps = ["JSON", "Suppressor"] +git-tree-sha1 = "ca2f4bae4aa6a99f81aac1075f949a6a4d392443" +uuid = "770da0de-323d-4d28-9202-0e205c1e0aff" +version = "0.4.4" + +[[deps.VectorizationBase]] +deps = ["ArrayInterface", "CPUSummary", "HostCPUFeatures", "IfElse", "LayoutPointers", "Libdl", "LinearAlgebra", "SIMDTypes", "Static", "StaticArrayInterface"] +git-tree-sha1 = "4ab62a49f1d8d9548a1c8d1a75e5f55cf196f64e" +uuid = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f" +version = "0.21.71" + +[[deps.VersionParsing]] +git-tree-sha1 = "58d6e80b4ee071f5efd07fda82cb9fbe17200868" +uuid = "81def892-9a0e-5fdd-b105-ffc91e053289" +version = "1.3.0" + +[[deps.WeakRefStrings]] +deps = ["DataAPI", "InlineStrings", "Parsers"] +git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" +uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" +version = "1.4.2" + +[[deps.WoodburyMatrices]] +deps = ["LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "5f24e158cf4cee437052371455fe361f526da062" +uuid = "efce3f68-66dc-5838-9240-27a6d6f5f9b6" +version = "0.5.6" + +[[deps.WorkerUtilities]] +git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7" +uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60" +version = "1.6.1" + +[[deps.XZ_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "15e637a697345f6743674f1322beefbc5dcd5cfc" +uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" +version = "5.6.3+0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+1" + +[[deps.Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "555d1076590a6cc2fdee2ef1469451f872d8b41b" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.6+1" + +[[deps.libaec_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "46bf7be2917b59b761247be3f317ddf75e50e997" +uuid = "477f73a3-ac25-53e9-8cc3-50b2fa2566f0" +version = "1.1.2+0" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.11.0+0" + +[[deps.libpng_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "b70c870239dc3d7bc094eb2d6be9b73d27bef280" +uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" +version = "1.6.44+0" + +[[deps.libsixel_jll]] +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Pkg", "libpng_jll"] +git-tree-sha1 = "7dfa0fd9c783d3d0cc43ea1af53d69ba45c447df" +uuid = "075b6546-f08a-558a-be8f-8157d0f608a5" +version = "1.10.3+1" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.52.0+1" + +[[deps.oneTBB_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "7d0ea0f4895ef2f5cb83645fa689e52cb55cf493" +uuid = "1317d2d5-d96f-522e-a858-c73665f53c3e" +version = "2021.12.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+2" diff --git a/Project.toml b/IFTPipeline.jl/Project.toml similarity index 60% rename from Project.toml rename to IFTPipeline.jl/Project.toml index b07a2a03..283960ed 100644 --- a/Project.toml +++ b/IFTPipeline.jl/Project.toml @@ -5,13 +5,20 @@ version = "0.1.0" [deps] ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +CSVFiles = "5d742f6a-9f54-50ce-8119-2520741973ca" +FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +FixedPointNumbers = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" Folds = "41a02a25-b8f0-4f67-bc48-60067656b558" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" IceFloeTracker = "04643c7a-9ac6-48c5-822f-2704f9e70bd3" +ImageSegmentation = "80713f31-8817-5129-9cf8-209ff8fb23e1" +Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" [compat] @@ -23,5 +30,6 @@ Pkg = "1.9" PyCall = "1.96" TOML = "1.0" -[source] -IceFloeTracker = {url = "https://github.com/WilhelmusLab/IceFloeTracker.jl", rev = "main"} +[sources.IceFloeTracker] +rev = "main" +url = "https://github.com/WilhelmusLab/IceFloeTracker.jl" diff --git a/IFTPipeline.jl/README.md b/IFTPipeline.jl/README.md new file mode 100644 index 00000000..24ddabda --- /dev/null +++ b/IFTPipeline.jl/README.md @@ -0,0 +1,37 @@ +# IFTPipeline.jl + +Command line interface for [IceFloeTracker.jl](https://github.com/WilhelmusLab/IceFloeTracker.jl) + +## Testing + +Run the tests in Julia: +```julia +julia> ] +(@v1.10) pkg> activate IFTPipeline.jl +(IFTPipeline) pkg> test + Testing IFTPipeline + ... +``` + +## Running the command line tools + +Use the help for wrapper scripts to learn about available options in each wrapper function. +For example: +``` +julia --project=IFTPipeline.jl ./src/cli.jl --help +``` + +## Debugging + +Debug messages (from `@debug` macros in the code) can be activated for +each command line task by calling Julia with the `JULIA_DEBUG` environment variable set. + +To activate debug logging for the Ice Floe Tracker, call: +```bash +JULIA_DEBUG="Main,IFTPipeline,IceFloeTracker" julia --project=IFTPipeline.jl IFTPipeline.jl/src/cli.jl ... +``` + +## Troubleshooting + +If you encouter errors with `PyCall` or `Conda`, follow the instructions in +[../PythonSetupForIFTPipeline.jl](../PythonSetupForIFTPipeline.jl/) to reinitialize the Conda environment. \ No newline at end of file diff --git a/src/IFTPipeline.jl b/IFTPipeline.jl/src/IFTPipeline.jl similarity index 77% rename from src/IFTPipeline.jl rename to IFTPipeline.jl/src/IFTPipeline.jl index cb9a8f2b..3a9c3977 100644 --- a/src/IFTPipeline.jl +++ b/IFTPipeline.jl/src/IFTPipeline.jl @@ -13,7 +13,13 @@ using Folds using HDF5 using TOML: parsefile using Pkg -include("cli-config.jl") +using FileIO +using Images +using CSV +using ImageSegmentation +using FixedPointNumbers + +include("cli.jl") include("soit-parser.jl") include("landmask.jl") include("preprocess.jl") @@ -24,6 +30,7 @@ include("h5.jl") export cache_vector, sharpen, sharpen_gray, preprocess, + preprocess_single, cloudmask, extractfeatures, get_ice_labels, @@ -32,10 +39,16 @@ export cache_vector, sharpen, load_truecolor_imgs, load_falsecolor_imgs, load_cloudmask, + load_labeled_img, + save_labeled_img, + convert_gray_from_uint, + convert_uint_from_gray, disc_ice_water, landmask, + landmask_single, track, mkclipreprocess!, + mkclipreprocess_single!, mkcliextract!, mkclitrack!, mkfilenames, diff --git a/IFTPipeline.jl/src/cli.jl b/IFTPipeline.jl/src/cli.jl new file mode 100755 index 00000000..6b8616c2 --- /dev/null +++ b/IFTPipeline.jl/src/cli.jl @@ -0,0 +1,541 @@ +#!/usr/bin/env julia + +using ArgParse +using IceFloeTracker +using IFTPipeline +using Serialization + +function mkclipreprocess!(settings) + @add_arg_table! settings["preprocess"] begin + "--truedir", "-t" + help = "Truecolor image directory" + required = true + + "--fcdir", "-r" + help = "Falsecolor image directory" + required = true + + "--lmdir", "-l" + help = "Land mask image directory" + required = true + + "--passtimesdir", "-p" + help = "Pass times directory" + required = true + + "--output", "-o" + help = "Output directory" + required = true + end + return nothing +end + +function mkclipreprocess_single!(settings) + @add_arg_table! settings["preprocess_single"] begin + "--truecolor", "-t" + help = "Truecolor image file (.tiff)" + required = true + + "--falsecolor", "-r" + help = "Falsecolor image file (.tiff)" + required = true + + "--landmask", "-l" + help = "Landmask image file (.tiff)" + required = true + + "--landmask-dilated", "-d" + help = "Landmask image file (dilated, .tiff)" + required = true + + "--output", "-o" + help = "Path to output segmented image file (.tiff)" + required = true + end + return nothing +end + +function mkcliextract!(settings) + @add_arg_table! settings["extractfeatures"] begin + "--input", "-i" + help = "Input image directory" + required = true + + "--output", "-o" + help = "Output image directory" + required = true + + "--minarea" + help = "Minimum area (in pixels) of ice floes to extract" + default = "350" + + "--maxarea" + help = "Maximum area (in pixels) of ice floes to extract" + default = "90000" + + "--features", "-f" + help = """Features to extract. Format: "feature1 feature2". For an extensive list of extractable features see https://scikit-image.org/docs/stable/api/skimage.measure.html#skimage.measure.regionprops:~:text=The%20following%20properties%20can%20be%20accessed%20as%20attributes%20or%20keys""" + default = "centroid area major_axis_length minor_axis_length convex_area bbox orientation perimeter" + end + return nothing +end + +function mkcliextract_single!(settings) + @add_arg_table! settings["extractfeatures_single"] begin + "--input", "-i" + help = "Input image" + required = true + + "--output", "-o" + help = "Output file (csv)" + required = true + + "--minarea" + help = "Minimum area (in pixels) of ice floes to extract" + arg_type = Int + default = 350 + + "--maxarea" + help = "Maximum area (in pixels) of ice floes to extract" + arg_type = Int + default = 90000 + + "--features", "-f" + help = """Features to extract. Format: "feature1 feature2". For an extensive list of extractable features see https://scikit-image.org/docs/stable/api/skimage.measure.html#skimage.measure.regionprops:~:text=The%20following%20properties%20can%20be%20accessed%20as%20attributes%20or%20keys""" + nargs = '+' + arg_type = String + default = ["label", "centroid", "area", "major_axis_length", "minor_axis_length", "convex_area", "bbox", "orientation", "perimeter"] + end + return nothing +end + +function mkclimakeh5!(settings) + @add_arg_table! settings["makeh5files"] begin + "--pathtosampleimg", "-p" + help = "Path to a sample image with coordinate reference system (CRS) and latitude and longitude coordinates of image pixels" + arg_type = String + + "--resdir", "-r" + help = "Path to the directory containing serialized results of the IceFloeTracker pipeline" + arg_type = String + end + return nothing +end + +function mkclimakeh5_single!(settings) + @add_arg_table! settings["makeh5files_single"] begin + "--iftversion" + help = "Version number of the IceFloeTracker.jl package" + required = false + arg_type = String + + "--passtime" + help = "Satellite pass time" + required = true + arg_type = DateTime + + "--truecolor" + help = "Path to truecolor image" + required = true + arg_type = String + + "--falsecolor" + help = "Path to falsecolor image" + required = true + arg_type = String + + "--labeled" + help = "Path to labeled image" + required = true + arg_type = String + + "--props" + help = "Path to extracted features (csv)" + required = true + arg_type = String + + "--output", "-o" + help = "Output file" + required = true + end + return nothing +end + +function mkclitrack!(settings) + add_arg_group!(settings["track"], "arguments") + @add_arg_table! settings["track"] begin + "--imgs" + help = "Path to object with segmented images" + required = true + + "--props" + help = "Path to object with extracted features" + required = true + + "--passtimes" + help = "Path to object with satellite pass times" + required = true + + "--latlon" + help = "Path to geotiff image with latitude/longitude data" + required = true + + "--output", "-o" + help = "Output directory" + required = true + end + + add_arg_group!(settings["track"], "optional arguments") + @add_arg_table! settings["track"] begin + "--params" + help = "Path to TOML file with algorithm parameters" + + "--area" + help = "Area thresholds to use for pairing floes" + arg_type = Int64 + default = 1200 + + "--dist" + help = "Distance threholds to use for pairing floes" + default = "15 30 120" + + "--dt-thresh" + help = "Time thresholds to use for pairing floes" + default = "30 100 1300" + + "--Sarearatio" + help = "Area ratio threshold for small floes" + arg_type = Float64 + default = 0.18 + + "--Smajaxisratio" + help = "Major axis ratio threshold for small floes" + arg_type = Float64 + default = 0.1 + + "--Sminaxisratio" + help = "Minor axis ratio threshold for small floes" + arg_type = Float64 + default = 0.12 + + "--Sconvexarearatio" + help = "Convex area ratio threshold for small floes" + arg_type = Float64 + default = 0.14 + + "--Larearatio" + help = "Area ratio threshold for large floes" + arg_type = Float64 + default = 0.28 + + "--Lmajaxisratio" + help = "Major axis ratio threshold for large floes" + arg_type = Float64 + default = 0.1 + + "--Lminaxisratio" + help = "Minor axis ratio threshold for large floes" + arg_type = Float64 + default = 0.15 + + "--Lconvexarearatio" + help = "Convex area ratio threshold for large floes" + arg_type = Float64 + default = 0.14 + + # matchcorr computation + "--mxrot" + help = "Maximum rotation" + arg_type = Int64 + default = 10 + + "--psi" + help = "Minimum psi-s correlation" + arg_type = Float64 + default = 0.95 + + "--sz" + help = "Minimum side length of floe mask" + arg_type = Int64 + default = 16 + + "--comp" + help = "Size comparability" + arg_type = Float64 + default = 0.25 + + "--mm" + help = "Maximum registration mismatch" + arg_type = Float64 + default = 0.22 + + # Goodness of match + "--corr" + help = "Mininimun psi-s correlation" + arg_type = Float64 + default = 0.68 + + "--area2" + help = "Large floe area mismatch threshold" + arg_type = Float64 + default = 0.236 + + "--area3" + help = "Small floe area mismatch threshold" + arg_type = Float64 + default = 0.18 + end + return nothing +end + +function mkclitrack_single!(settings) + add_arg_group!(settings["track_single"], "arguments") + @add_arg_table! settings["track_single"] begin + "--imgs" + help = "Paths to segmented images" + required = true + nargs = '+' + arg_type = String + + "--props" + help = "Paths to extracted features" + required = true + nargs = '+' + arg_type = String + + "--passtimes" + help = "Path to object with satellite pass times" + required = true + nargs = '+' + arg_type = DateTime + + "--latlon" + help = "Path to geotiff image with latitude/longitude data" + required = true + + "--output", "-o" + help = "Output file" + required = true + end + + add_arg_group!(settings["track_single"], "optional arguments") + @add_arg_table! settings["track_single"] begin + + "--area" + help = "Area thresholds to use for pairing floes" + arg_type = Int64 + default = 1200 + + "--dist" + help = "Distance threholds to use for pairing floes" + default = [15, 30, 120] + arg_type = Int + nargs = '+' + + "--dt-thresh" + help = "Time thresholds to use for pairing floes" + default = [30, 100, 1300] + arg_type = Int + nargs = '+' + + "--Sarearatio" + help = "Area ratio threshold for small floes" + arg_type = Float64 + default = 0.18 + + "--Smajaxisratio" + help = "Major axis ratio threshold for small floes" + arg_type = Float64 + default = 0.1 + + "--Sminaxisratio" + help = "Minor axis ratio threshold for small floes" + arg_type = Float64 + default = 0.12 + + "--Sconvexarearatio" + help = "Convex area ratio threshold for small floes" + arg_type = Float64 + default = 0.14 + + "--Larearatio" + help = "Area ratio threshold for large floes" + arg_type = Float64 + default = 0.28 + + "--Lmajaxisratio" + help = "Major axis ratio threshold for large floes" + arg_type = Float64 + default = 0.1 + + "--Lminaxisratio" + help = "Minor axis ratio threshold for large floes" + arg_type = Float64 + default = 0.15 + + "--Lconvexarearatio" + help = "Convex area ratio threshold for large floes" + arg_type = Float64 + default = 0.14 + + # matchcorr computation + "--mxrot" + help = "Maximum rotation" + arg_type = Int64 + default = 10 + + "--psi" + help = "Minimum psi-s correlation" + arg_type = Float64 + default = 0.95 + + "--sz" + help = "Minimum side length of floe mask" + arg_type = Int64 + default = 16 + + "--comp" + help = "Size comparability" + arg_type = Float64 + default = 0.25 + + "--mm" + help = "Maximum registration mismatch" + arg_type = Float64 + default = 0.22 + + # Goodness of match + "--corr" + help = "Mininimun psi-s correlation" + arg_type = Float64 + default = 0.68 + + "--area2" + help = "Large floe area mismatch threshold" + arg_type = Float64 + default = 0.236 + + "--area3" + help = "Small floe area mismatch threshold" + arg_type = Float64 + default = 0.18 + end + return nothing +end + +function mkclilandmask!(settings) + args = [ + "input", + Dict(:help => "Input image directory", :required => true), + "output", + Dict(:help => "Output image directory", :required => true), + ] + add_arg_table!(settings["landmask"], args...) +end + +function mkclilandmask_single!(settings) + @add_arg_table! settings["landmask_single"] begin + "--input", "-i" + help = "Input image" + required = true + + "--output_non_dilated", "-o" + help = "Output path for binarized landmask" + required = true + + "--output_dilated", "-d" + help = "Output path for dilated, binarized landmask" + required = true + end + return nothing +end + +function mkcli!(settings, common_args) + d = Dict( + "landmask" => mkclilandmask!, + "landmask_single" => mkclilandmask_single!, + "preprocess" => mkclipreprocess!, + "preprocess_single" => mkclipreprocess_single!, + "extractfeatures" => mkcliextract!, + "extractfeatures_single" => mkcliextract_single!, + "makeh5files" => mkclimakeh5!, + "makeh5files_single" => mkclimakeh5_single!, + "track" => mkclitrack!, + "track_single" => mkclitrack_single!, + ) + + for t in keys(d) + d[t](settings) # add arguments to settings + add_arg_table!(settings[t], common_args...) + end + return nothing +end + + +function main() + + settings = ArgParseSettings(; autofix_names=true) + + @add_arg_table! settings begin + "landmask" + help = "Generate land mask images" + action = :command + + "landmask_single" + help = "Generate land mask images" + action = :command + + "preprocess" + help = "Preprocess truecolor/falsecolor images" + action = :command + + "preprocess_single" + help = "Preprocess truecolor/falsecolor images" + action = :command + + "extractfeatures" + help = "Extract ice floe features from segmented floe image" + action = :command + + "extractfeatures_single" + help = "Extract ice floe features from a single segmented floe image" + action = :command + + "makeh5files" + help = "Make HDF5 files from extracted floe features" + action = :command + + "makeh5files_single" + help = "Make HDF5 files from extracted floe features" + action = :command + + "track" + help = "Pair ice floes in day k with ice floes in day k+1" + action = :command + + "track_single" + help = "Pair ice floes in day k with ice floes in day k+1" + action = :command + end + + command_common_args = [] + + + mkcli!(settings, command_common_args) + + parsed_args = parse_args(settings; as_symbols=true) + + command = parsed_args[:_COMMAND_] + command_args = parsed_args[command] + command_func = getfield(IFTPipeline, Symbol(command)) + + @debug "debug message" + @info "info message" + @time command_func(; command_args...) + + return nothing +end + +if abspath(PROGRAM_FILE) == @__FILE__ + main() +end diff --git a/src/feature-extraction.jl b/IFTPipeline.jl/src/feature-extraction.jl similarity index 63% rename from src/feature-extraction.jl rename to IFTPipeline.jl/src/feature-extraction.jl index 515cfdaa..60e97f1b 100644 --- a/src/feature-extraction.jl +++ b/IFTPipeline.jl/src/feature-extraction.jl @@ -48,16 +48,18 @@ julia> IFTPipeline.extractfeatures(bw_img; minarea=minarea, maxarea=maxarea, fea ``` """ function extractfeatures( - bw::T; + floes::AbstractArray{<:Union{Integer,Bool}}; minarea::Int64=350, maxarea::Int64=90000, features::Union{Vector{Symbol},Vector{<:AbstractString}} -)::DataFrame where {T<:AbstractArray{Bool}} +)::DataFrame # assert the first area threshold is less than the second minarea >= maxarea && throw(ArgumentError("The minimum area must be less than the maximum area.")) - props = regionprops_table(label_components(bw, trues(3, 3)); properties=features) + floes = (eltype(floes) <: Bool ? label_components(floes, trues(3, 3)) : floes) + props = regionprops_table(floes; properties=features) + @debug "loaded $props" # filter by area using the area thresholds return props[minarea.<=props.area.<=maxarea, :] @@ -90,3 +92,92 @@ function extractfeatures(; serialize(joinpath(output, "floe_props.jls"), props) return nothing end + +function extractfeatures_single(; + input::String, output::String, minarea::Int64, maxarea::Int64, features::Array{String} +) + @info "Loading segmented floes from $input" + labeled_floes = Int.(load_labeled_img(input)) + + @info "Extracting features from each floe: $features" + props = IFTPipeline.extractfeatures(labeled_floes; minarea=minarea, maxarea=maxarea, features=features) + + @info "Extracted properties:" + @info props + + @info "Writing to $output" + FileIO.save(output, props) + return nothing +end + +""" + load_labeled_img(path) + +Load an unsigned integer image from a file. + +See also: save_labeled_img +""" +function load_labeled_img(path::AbstractString) + image = FileIO.load(path) + image_reinterpreted = convert_uint_from_gray(image) + return image_reinterpreted +end + +""" + save_labeled_img(image, path) + +Save an unsigned integer image to an image file. + +See also: load_labeled_img +""" +function save_labeled_img(image::AbstractArray{<:Integer}, path::AbstractString) + image_reinterpreted = convert_gray_from_uint(image) + FileIO.save(path, image_reinterpreted) + return path +end + +INT_TO_FIXEDPOINT_MAP = Dict( + UInt8 => N0f8, + Int8 => Q0f7, + UInt16 => N0f16, + Int16 => Q0f15, + UInt32 => N0f32, + Int32 => Q0f31, + UInt64 => N0f64, + Int64 => Q0f63 +) + +""" + convert_gray_from_uint(image) + +Convert an image from an unsigned integer format into a fixed-point Gray format. + +See also: convert_uint_from_gray +""" +function convert_gray_from_uint(image::AbstractArray{<:Integer}) + img_type = eltype(image) + target_type = get(INT_TO_FIXEDPOINT_MAP, img_type) do + error("Missing mapping for $img_type in convert_gray_from_uint") + end + image_reinterpreted = Gray.(reinterpret.(target_type, image)) + return image_reinterpreted +end + +FIXEDPOINT_TO_INT_MAP = Dict(Gray{value} => key for (key, value) in INT_TO_FIXEDPOINT_MAP) + +""" + convert_uint_from_gray(image) + +Convert an image from a fixed-point Gray format into integers. + +See also: convert_gray_from_uint +""" +function convert_uint_from_gray(image) + image_reinterpreted = rawview(channelview(image)) + img_type = eltype(image) + target_type = get(FIXEDPOINT_TO_INT_MAP, img_type) do + error("Missing mapping for $img_type in convert_uint_from_gray") + end + image_recast = target_type.(image_reinterpreted) + return image_recast +end \ No newline at end of file diff --git a/src/h5.jl b/IFTPipeline.jl/src/h5.jl similarity index 69% rename from src/h5.jl rename to IFTPipeline.jl/src/h5.jl index 7b501fd9..7e100862 100644 --- a/src/h5.jl +++ b/IFTPipeline.jl/src/h5.jl @@ -26,7 +26,7 @@ Choose the appropriate unsigned integer type based on a maximum value. * `UInt128` if 0 ≤ `mx` ≤ 2^128 - 1. """ -function choose_dtype(mx::T) where {T <: Integer} +function choose_dtype(mx::T) where {T<:Integer} types = [UInt8, UInt16, UInt32, UInt64, UInt128] for t_ in types if typemin(t_) <= mx <= typemax(t_) @@ -136,3 +136,53 @@ function makeh5files(; pathtosampleimg::String, resdir::String, iftversion=IceFl end return nothing end + + +function makeh5files_single(; passtime::DateTime, iftversion::Union{String,Nothing}=nothing, truecolor::String, falsecolor::String, labeled::String, props::String, output::String) + latlondata = getlatlon(truecolor) + ptsunix = Int64(Dates.datetime2unix(passtime)) + labeled_ = load_labeled_img(labeled) + + if isnothing(iftversion) + iftversion = string(IceFloeTracker.IFTVERSION) + end + + props_ = DataFrame(CSV.File(props)) + colstodrop = [:row_centroid, :col_centroid, :min_row, :min_col, :max_row, :max_col] + converttounits!(props_, latlondata, colstodrop) + @info props_ + + h5open(output, "w") do file + # Add top-level attributes + attrs(file)["fname_falsecolor"] = falsecolor + attrs(file)["fname_truecolor"] = truecolor + attrs(file)["iftversion"] = iftversion + attrs(file)["crs"] = latlondata["crs"] + attrs(file)["reference"] = "https://doi.org/10.1016/j.rse.2019.111406" + attrs(file)["contact"] = "mmwilhelmus@brown.edu" + + g = create_group(file, "index") + g["time"] = ptsunix + g["x"] = latlondata["X"] + g["y"] = latlondata["Y"] + + g = create_group(file, "floe_properties") + write_dataset(g, "properties", [copy(row) for row in eachrow(props_)]) # `copy(row)` converts the DataSetRow to a NamedTuple + attrs(g)["Description of properties"] = """ Area units (`area`, `convex_area`) are in sq. kilometers, length units (`minor_axis_length`, `major_axis_length`, and `perimeter`) in kilometers, and `orientation` in radians (see the description of properties attribute.) Latitude and longitude coordinates are in degrees, and the stereographic coordinates`x` and `y` are in meters relative to the NSIDC north polar stereographic projection. Generated using the `regionprops` function from the `skimage` package. See https://scikit-image.org/docs/0.20.x/api/skimage.measure.html#regionprops + """ + + mx = maximum(labeled_) + T = choose_dtype(mx) + # write_dataset(g, "labeled_image", T.(labeled_)) + imgdata = T.(permutedims(labeled_)) + obj, dtype = create_dataset(g, "labeled_image", imgdata) + attrs(obj)["CLASS"] = "IMAGE" + attrs(obj)["IMAGE_SUBCLASS"] = "IMAGE_INDEXED" + attrs(obj)["IMAGE_MINMAXRANGE"] = [minimum(imgdata), maximum(imgdata)] + + attrs(obj)["description"] = "Connected components of the segmented floe image using a 3x3 structuring element. The property matrix consists of the properties of each connected component." + write_dataset(obj, dtype, imgdata) + + end + return nothing +end \ No newline at end of file diff --git a/src/landmask.jl b/IFTPipeline.jl/src/landmask.jl similarity index 53% rename from src/landmask.jl rename to IFTPipeline.jl/src/landmask.jl index 4569194f..3f40b6a2 100644 --- a/src/landmask.jl +++ b/IFTPipeline.jl/src/landmask.jl @@ -24,3 +24,32 @@ function landmask(; input::String, output::String, landmask_fname::String="landm @info "Landmask created and serialized succesfully." return nothing end + + +""" + landmask_single(; input, output_non_dilated, output_dilated) + + Given an input directory with a landmask file, create a land/soft ice mask object with both dilated and non_dilated versions. The object is serialized to the output directory. + + # Arguments +- `input`: path to landmask source image +- `output_non_dilated`: path to output file for non-dilated landmask +- `output_dilated`: path to output file for dilated landmask +""" +function landmask_single(; input::String, output_non_dilated::String, output_dilated::String) + @info "Using $input as landmask" + img = load(input) + @info "Landmask found at $input." + + @info "Create landmask, both dilated and non-dilated" + landmask = create_landmask(img) + + @info "Saving non-dilated landmask to $output_non_dilated" + FileIO.save(output_non_dilated, landmask.non_dilated) + + @info "Saving dilated landmask to $output_dilated" + FileIO.save(output_dilated, landmask.dilated) + + @info "Landmask created and serialized succesfully." + return nothing +end diff --git a/src/preprocess.jl b/IFTPipeline.jl/src/preprocess.jl similarity index 85% rename from src/preprocess.jl rename to IFTPipeline.jl/src/preprocess.jl index 5b873c33..d3394dd9 100644 --- a/src/preprocess.jl +++ b/IFTPipeline.jl/src/preprocess.jl @@ -256,3 +256,45 @@ function preprocess(; truedir::T, fcdir::T, lmdir::T, passtimesdir::T, output::T serialize(joinpath(output, "passtimes.jls"), soitdf.pass_time) return nothing end + +""" + preprocess(; truecolor::T, falsecolor::T, landmask::T, landmask_dilated::T, output::T) where {T<:AbstractString} + +Preprocess and segment floes in a single view. Save the segmented floes to `output`. + +# Arguments +- `truecolor`: path to truecolor image +- `falsecolor`: path to falsecolor image +- `landmask`: path to landmask image +- `landmask_dilated`: path to dilated landmask image +- `output`: path to output file +""" +function preprocess_single(; truecolor::T, falsecolor::T, landmask::T, landmask_dilated::T, output::T) where {T<:AbstractString} + + @info "Processing images: $truecolor, $falsecolor, $landmask" + truecolor_img = loadimg(; dir=dirname(truecolor), fname=basename(truecolor)) + falsecolor_img = loadimg(; dir=dirname(falsecolor), fname=basename(falsecolor)) + + # TODO: make symmetric landmask saving/loading functions + landmask = ( + dilated=BitMatrix(FileIO.load(landmask_dilated)), + non_dilated=BitMatrix(FileIO.load(landmask)), + ) + + @info "Removing alpha channel if it exists" + rgb_truecolor_img = RGB.(truecolor_img) + rgb_falsecolor_img = RGB.(falsecolor_img) + + @info "Segmenting floes" + segmented_floes = preprocess(rgb_truecolor_img, rgb_falsecolor_img, landmask) + + @info "Labeling floes" + labeled_floes = label_components(segmented_floes) + _dtype = choose_dtype(maximum(labeled_floes)) + labeled_floes_cast = convert(Array{_dtype}, labeled_floes) + + @info "Writing segmented floes to $output" + save_labeled_img(labeled_floes_cast, output) + + return nothing +end \ No newline at end of file diff --git a/src/soit-parser.jl b/IFTPipeline.jl/src/soit-parser.jl similarity index 100% rename from src/soit-parser.jl rename to IFTPipeline.jl/src/soit-parser.jl diff --git a/IFTPipeline.jl/src/tracker.jl b/IFTPipeline.jl/src/tracker.jl new file mode 100644 index 00000000..fc9b7db3 --- /dev/null +++ b/IFTPipeline.jl/src/tracker.jl @@ -0,0 +1,207 @@ +""" + track( + imgsdir::String, + propsdir::String, + passtimesdir::String, + paramsdir::String, + outdir::String, +) + +Pair floes in the floe library using an equivalent implementation as in the MATLAB script `final_2020.m` from https://github.com/WilhelmusLab/ice-floe-tracker/blob/main/existing_code/final_2020.m. + +# Arguments +- `indir`: path to directory containing the floe library (cropped floe masks for registration and correlation), extracted features from segmented images, and satellite overpass times. +- `condition_thresholds`: 3-tuple of thresholds (each a named tuple) for deciding whether to match floe `i` from day `k` to floe j from day `k+1`. +- `mc_thresholds`: thresholds for area mismatch and psi-s shape correlation and goodness of a match. + +See `IceFloeTracker.jl/notebooks/track-floes/track-floes.ipynb` for a sample workflow. + +Following are the default set of thresholds `condition_thresholds` used for floe matching: +- Condition 1: time elapsed `dt` from image `k` to image `k+1` and distance between floes centroids `dist`: `t1=(dt = (30, 100, 1300), dist=(15, 30, 120))` + +- Condition 2: area of floe i `area`, and the computed ratios for area, major axis, minor axis, and convex area of floes `i` and `j` in days `k` and `k+1`, respectively: `t2=(area=1200, arearatio=0.28, majaxisratio=0.10, minaxisratio=0.12, convexarearatio=0.14)` + +- Condition 3: as in the previous condition but set as follows: ` +t3=(area=1200, arearatio=0.18, majaxisratio=0.07, minaxisratio=0.08, convexarearatio=0.09)` +""" +function track(; args...) + condition_thresholds, mc_thresholds = get_thresholds(; args...) + vals = NamedTuple{Tuple(keys(args))}(values(args)) # convert to NamedTuple + imgs = deserialize(joinpath(vals.imgs, "segmented_floes.jls")) + props = deserialize(joinpath(vals.props, "floe_props.jls")) + passtimes = deserialize(joinpath(vals.passtimes, "passtimes.jls")) + latlon = vals.latlon + labeledfloes = IceFloeTracker.pairfloes(imgs, props, passtimes, latlon, condition_thresholds, mc_thresholds) + serialize(joinpath(vals.output, "labeled_floes.jls"), labeledfloes) + return nothing +end + +""" + track_single( + imgs::Array{String}, + props::Array{String}, + passtimes::Array{DateTime}, + latlon::String, + output::String, + ... +) +Pair floes in the floe library using an equivalent implementation as in the MATLAB script `final_2020.m` from https://github.com/WilhelmusLab/ice-floe-tracker/blob/main/existing_code/final_2020.m. + +# Arguments +- `imgs`: segmented images filenames (.tiff) +- `props`: corresponding floe properties filenames (.csv) +- `passtimes`: corresponding pass-times for each image +- `latlon`: an example image with geospatial data (e.g., the original true-color image, .tiff) +- `output`: filename for output file (.csv) +- tracker parameters (see source code) + +Following are the default set of thresholds `condition_thresholds` used for floe matching: +- Condition 1: time elapsed `dt` from image `k` to image `k+1` and distance between floes centroids `dist`: `t1=(dt = (30, 100, 1300), dist=(15, 30, 120))` +- Condition 2: area of floe i `area`, and the computed ratios for area, major axis, minor axis, and convex area of floes `i` and `j` in days `k` and `k+1`, respectively: `t2=(area=1200, arearatio=0.28, majaxisratio=0.10, minaxisratio=0.12, convexarearatio=0.14)` +- Condition 3: as in the previous condition but set as follows: `t3=(area=1200, arearatio=0.18, majaxisratio=0.07, minaxisratio=0.08, convexarearatio=0.09)` +""" +function track_single(; + imgs::Array{String}, + props::Array{String}, + passtimes::Array{DateTime}, + latlon::String, + output::String, + area::Int64=1200, + dist::Array{Int}=[15, 30, 120], + dt_thresh::Array{Int}=[30, 100, 1300], + Sarearatio::Float64=0.18, + Smajaxisratio::Float64=0.1, + Sminaxisratio::Float64=0.12, + Sconvexarearatio::Float64=0.14, + Larearatio::Float64=0.28, + Lmajaxisratio::Float64=0.1, + Lminaxisratio::Float64=0.15, + Lconvexarearatio::Float64=0.14, + mxrot::Int64=10, + psi::Float64=0.95, + sz::Int64=16, + comp::Float64=0.25, + mm::Float64=0.22, + corr::Float64=0.68, + area2::Float64=0.236, + area3::Float64=0.18, +) + + # Load the files – can we drop the memory requirements by doing two at once? + @info "Loading $imgs" + imgs_ = [load_labeled_img(img) for img in imgs] + + @info "Loading $props" + props_ = [DataFrame(CSV.File(prop)) for prop in props] + # go through each of the props_ dataframes and convert each + # into the element type from the corresponding image. + for (img_, prop_) in zip(imgs_, props_) + label_type = eltype(img_) + @debug "converting labels to $label_type" + prop_[!,:label] = convert.(label_type, prop_[!,:label]) + end + @info "Loaded: $props_" + + @info "Using passtimes=$passtimes" + + condition_thresholds = ( + t1=( + dt=dt_thresh, + dist=dist + ), + t2=( + area=area, + arearatio=Larearatio, + convexarearatio=Lconvexarearatio, + majaxisratio=Lmajaxisratio, + minaxisratio=Lminaxisratio, + ), + t3=( + area=area, + arearatio=Sarearatio, + convexarearatio=Sconvexarearatio, + majaxisratio=Smajaxisratio, + minaxisratio=Sminaxisratio, + ), + ) + + mc_thresholds = ( + comp=( + mxrot=mxrot, + sz=sz, + comp=comp, + mm=mm, + psi=psi + ), + goodness=( + corr=corr, + area2=area2, + area3=area3 + ), + ) + + labeled_floes = IceFloeTracker.pairfloes(imgs_, props_, passtimes, latlon, condition_thresholds, mc_thresholds) + FileIO.save(output, labeled_floes) + return nothing +end + +function parse_params(params::AbstractString) + params = parsefile(params) + area = params["area"] + t1 = dict2nt(params["t1"]) + t2 = (area=area, dict2nt(params["t2"])...) + t3 = (area=area, dict2nt(params["t3"])...) + condition_thresholds = (t1=t1, t2=t2, t3=t3) + d = dict2nt(params["mc_thresholds"]) + mc_thresholds = mkmct(d) + return condition_thresholds, mc_thresholds +end + +function parse_params(; args...) + d = values(args) + condition_thresholds = ( + t1=(dt=parselistas(Int64, d.dt_thresh), dist=parselistas(Int64, d.dist)), + t2=( + area=d.area, + arearatio=d.Larearatio, + convexarearatio=d.Lconvexarearatio, + majaxisratio=d.Lmajaxisratio, + minaxisratio=d.Lminaxisratio, + ), + t3=( + area=d.area, + arearatio=d.Sarearatio, + convexarearatio=d.Sconvexarearatio, + majaxisratio=d.Smajaxisratio, + minaxisratio=d.Sminaxisratio, + ), + ) + mc_thresholds = mkmct(d) + return condition_thresholds, mc_thresholds +end + +function mkmct(d) + return ( + comp=(mxrot=d.mxrot, sz=d.sz, comp=d.comp, mm=d.mm, psi=d.psi), + goodness=(corr=d.corr, area2=d.area2, area3=d.area3), + ) +end + +function get_thresholds(; args...) + v = values(args) + if !isnothing(v.params) + condition_thresholds, mc_thresholds = parse_params(v.params) + else + condition_thresholds, mc_thresholds = parse_params(; args...) + end + return condition_thresholds, mc_thresholds +end + +""" + dict2nt(d) + +Convert a dictionary `d` to a NamedTuple. +""" +dict2nt(d) = NamedTuple((Symbol(key), value) for (key, value) in d) + +parselistas(T, x) = [parse(T, i) for i in split(x)] diff --git a/IFTPipeline.jl/test/.gitignore b/IFTPipeline.jl/test/.gitignore new file mode 100644 index 00000000..da3f9f3b --- /dev/null +++ b/IFTPipeline.jl/test/.gitignore @@ -0,0 +1,3 @@ +__temp__ +test_inputs/input_pipeline/*.jls +!test_inputs/input_pipeline/{filenames,generated_landmask,passtimes}.jls diff --git a/test/Project.toml b/IFTPipeline.jl/test/Project.toml similarity index 100% rename from test/Project.toml rename to IFTPipeline.jl/test/Project.toml diff --git a/test/_test-preprocess.jl b/IFTPipeline.jl/test/_test-preprocess.jl similarity index 100% rename from test/_test-preprocess.jl rename to IFTPipeline.jl/test/_test-preprocess.jl diff --git a/test/_test-track.jl b/IFTPipeline.jl/test/_test-track.jl similarity index 96% rename from test/_test-track.jl rename to IFTPipeline.jl/test/_test-track.jl index c9a2e52d..cd4ccbe0 100644 --- a/test/_test-track.jl +++ b/IFTPipeline.jl/test/_test-track.jl @@ -6,7 +6,7 @@ settings = ArgParseSettings(; autofix_names=true) end IFTPipeline.mkclitrack!(settings) -params_path = "../config/sample-tracker-params.toml" +params_path = "test_inputs/sample-tracker-params.toml" latlonrefimage = "test_inputs/NE_Greenland_truecolor.2020162.aqua.250m.tiff" data_path = "test_inputs/tracker/" temp = mkpath(joinpath(@__DIR__, "__temp__")) diff --git a/test/config.jl b/IFTPipeline.jl/test/config.jl similarity index 100% rename from test/config.jl rename to IFTPipeline.jl/test/config.jl diff --git a/test/runtests.jl b/IFTPipeline.jl/test/runtests.jl similarity index 92% rename from test/runtests.jl rename to IFTPipeline.jl/test/runtests.jl index 0836f90d..39ebbe55 100644 --- a/test/runtests.jl +++ b/IFTPipeline.jl/test/runtests.jl @@ -28,12 +28,13 @@ testnames = [n[6:(end-3)] for n in alltests] ## Put the filenames to test below to_test = alltests # uncomment this line to run all tests or add individual files below -[ - "test-h5.jl" -] +# to_test = [ +# "test-h5.jl", +# "test-pipeline.jl" +# ] # Run the tests -@testset verbose = true "IceFloeTracker.jl" begin +@testset verbose = true "IFTPipeline.jl" begin for test in to_test include(test) end diff --git a/IFTPipeline.jl/test/test-feature-extract.jl b/IFTPipeline.jl/test/test-feature-extract.jl new file mode 100644 index 00000000..91e5d363 --- /dev/null +++ b/IFTPipeline.jl/test/test-feature-extract.jl @@ -0,0 +1,54 @@ +using Images +using IFTPipeline: load_labeled_img, save_labeled_img + +function test_save_load(image; extension=".tiff") + + filename = tempname() * extension + + saved_image = save_labeled_img(image, filename) + loaded_image = load_labeled_img(saved_image) + + @test isequal(image, loaded_image) +end + + +function test_cast_uncast(image) + + casted_image = convert_gray_from_uint(image) + uncasted_image = convert_uint_from_gray(casted_image) + + @test isequal(image, uncasted_image) + @test isequal(eltype(image), eltype(uncasted_image)) +end + + +@testset "feature-extraction.jl" begin + image_size = (8, 8) + + # Can cast and uncast all sizes of unsigned integer + _test_cast_uncast(t) = test_cast_uncast(rand(t, image_size)) + _test_cast_uncast(UInt8) + _test_cast_uncast(UInt16) + _test_cast_uncast(UInt32) + _test_cast_uncast(UInt64) + + # ... and signed integer + _test_cast_uncast(Int8) + _test_cast_uncast(Int16) + _test_cast_uncast(Int32) + _test_cast_uncast(Int64) + + # Can save and load all sizes of unsigned integer: + _test_save_load(t) = test_save_load(rand(t, image_size)) + _test_save_load(UInt8) + _test_save_load(UInt16) + _test_save_load(UInt32) + _test_save_load(UInt64) + + # ... and signed integer + _test_save_load(Int8) + _test_save_load(Int16) + _test_save_load(Int32) + _test_save_load(Int64) + +end \ No newline at end of file diff --git a/test/test-h5.jl b/IFTPipeline.jl/test/test-h5.jl similarity index 100% rename from test/test-h5.jl rename to IFTPipeline.jl/test/test-h5.jl diff --git a/test/test-pipeline.jl b/IFTPipeline.jl/test/test-pipeline.jl similarity index 100% rename from test/test-pipeline.jl rename to IFTPipeline.jl/test/test-pipeline.jl diff --git a/test/test_inputs/NE_Greenland_falsecolor.2020162.aqua.250m.tiff b/IFTPipeline.jl/test/test_inputs/NE_Greenland_falsecolor.2020162.aqua.250m.tiff similarity index 100% rename from test/test_inputs/NE_Greenland_falsecolor.2020162.aqua.250m.tiff rename to IFTPipeline.jl/test/test_inputs/NE_Greenland_falsecolor.2020162.aqua.250m.tiff diff --git a/test/test_inputs/NE_Greenland_truecolor.2020162.aqua.250m.tiff b/IFTPipeline.jl/test/test_inputs/NE_Greenland_truecolor.2020162.aqua.250m.tiff similarity index 100% rename from test/test_inputs/NE_Greenland_truecolor.2020162.aqua.250m.tiff rename to IFTPipeline.jl/test/test_inputs/NE_Greenland_truecolor.2020162.aqua.250m.tiff diff --git a/test/test_inputs/expected/generated_landmask.png b/IFTPipeline.jl/test/test_inputs/expected/generated_landmask.png similarity index 100% rename from test/test_inputs/expected/generated_landmask.png rename to IFTPipeline.jl/test/test_inputs/expected/generated_landmask.png diff --git a/test/test_inputs/input_pipeline/20220914.aqua.falsecolor.250m.tiff b/IFTPipeline.jl/test/test_inputs/input_pipeline/20220914.aqua.falsecolor.250m.tiff similarity index 100% rename from test/test_inputs/input_pipeline/20220914.aqua.falsecolor.250m.tiff rename to IFTPipeline.jl/test/test_inputs/input_pipeline/20220914.aqua.falsecolor.250m.tiff diff --git a/test/test_inputs/input_pipeline/20220914.aqua.truecolor.250m.tiff b/IFTPipeline.jl/test/test_inputs/input_pipeline/20220914.aqua.truecolor.250m.tiff similarity index 100% rename from test/test_inputs/input_pipeline/20220914.aqua.truecolor.250m.tiff rename to IFTPipeline.jl/test/test_inputs/input_pipeline/20220914.aqua.truecolor.250m.tiff diff --git a/test/test_inputs/input_pipeline/20220914.terra.falsecolor.250m.tiff b/IFTPipeline.jl/test/test_inputs/input_pipeline/20220914.terra.falsecolor.250m.tiff similarity index 100% rename from test/test_inputs/input_pipeline/20220914.terra.falsecolor.250m.tiff rename to IFTPipeline.jl/test/test_inputs/input_pipeline/20220914.terra.falsecolor.250m.tiff diff --git a/test/test_inputs/input_pipeline/20220914.terra.truecolor.250m.tiff b/IFTPipeline.jl/test/test_inputs/input_pipeline/20220914.terra.truecolor.250m.tiff similarity index 100% rename from test/test_inputs/input_pipeline/20220914.terra.truecolor.250m.tiff rename to IFTPipeline.jl/test/test_inputs/input_pipeline/20220914.terra.truecolor.250m.tiff diff --git a/IFTPipeline.jl/test/test_inputs/input_pipeline/filenames.jls b/IFTPipeline.jl/test/test_inputs/input_pipeline/filenames.jls new file mode 100644 index 00000000..1c83cec7 Binary files /dev/null and b/IFTPipeline.jl/test/test_inputs/input_pipeline/filenames.jls differ diff --git a/test/test_inputs/input_pipeline/generated_landmask.jls b/IFTPipeline.jl/test/test_inputs/input_pipeline/generated_landmask.jls similarity index 100% rename from test/test_inputs/input_pipeline/generated_landmask.jls rename to IFTPipeline.jl/test/test_inputs/input_pipeline/generated_landmask.jls diff --git a/test/test_inputs/input_pipeline/h5/filenames.jls b/IFTPipeline.jl/test/test_inputs/input_pipeline/h5/filenames.jls similarity index 100% rename from test/test_inputs/input_pipeline/h5/filenames.jls rename to IFTPipeline.jl/test/test_inputs/input_pipeline/h5/filenames.jls diff --git a/test/test_inputs/input_pipeline/h5/floe_props.jls b/IFTPipeline.jl/test/test_inputs/input_pipeline/h5/floe_props.jls similarity index 100% rename from test/test_inputs/input_pipeline/h5/floe_props.jls rename to IFTPipeline.jl/test/test_inputs/input_pipeline/h5/floe_props.jls diff --git a/test/test_inputs/input_pipeline/h5/passtimes.jls b/IFTPipeline.jl/test/test_inputs/input_pipeline/h5/passtimes.jls similarity index 100% rename from test/test_inputs/input_pipeline/h5/passtimes.jls rename to IFTPipeline.jl/test/test_inputs/input_pipeline/h5/passtimes.jls diff --git a/test/test_inputs/input_pipeline/h5/segmented_floes.jls b/IFTPipeline.jl/test/test_inputs/input_pipeline/h5/segmented_floes.jls similarity index 100% rename from test/test_inputs/input_pipeline/h5/segmented_floes.jls rename to IFTPipeline.jl/test/test_inputs/input_pipeline/h5/segmented_floes.jls diff --git a/test/test_inputs/input_pipeline/landmask.tiff b/IFTPipeline.jl/test/test_inputs/input_pipeline/landmask.tiff similarity index 100% rename from test/test_inputs/input_pipeline/landmask.tiff rename to IFTPipeline.jl/test/test_inputs/input_pipeline/landmask.tiff diff --git a/IFTPipeline.jl/test/test_inputs/input_pipeline/passtimes.jls b/IFTPipeline.jl/test/test_inputs/input_pipeline/passtimes.jls new file mode 100644 index 00000000..78ba88b3 Binary files /dev/null and b/IFTPipeline.jl/test/test_inputs/input_pipeline/passtimes.jls differ diff --git a/test/test_inputs/input_pipeline/passtimes_lat.csv b/IFTPipeline.jl/test/test_inputs/input_pipeline/passtimes_lat.csv similarity index 100% rename from test/test_inputs/input_pipeline/passtimes_lat.csv rename to IFTPipeline.jl/test/test_inputs/input_pipeline/passtimes_lat.csv diff --git a/test/test_inputs/matlab_BW7.png b/IFTPipeline.jl/test/test_inputs/matlab_BW7.png similarity index 100% rename from test/test_inputs/matlab_BW7.png rename to IFTPipeline.jl/test/test_inputs/matlab_BW7.png diff --git a/test/test_inputs/pipeline/feature_extraction/input/segmented_floes.jls b/IFTPipeline.jl/test/test_inputs/pipeline/feature_extraction/input/segmented_floes.jls similarity index 100% rename from test/test_inputs/pipeline/feature_extraction/input/segmented_floes.jls rename to IFTPipeline.jl/test/test_inputs/pipeline/feature_extraction/input/segmented_floes.jls diff --git a/config/sample-tracker-params.toml b/IFTPipeline.jl/test/test_inputs/sample-tracker-params.toml similarity index 100% rename from config/sample-tracker-params.toml rename to IFTPipeline.jl/test/test_inputs/sample-tracker-params.toml diff --git a/test/test_inputs/tracker/passtimes.dat b/IFTPipeline.jl/test/test_inputs/tracker/passtimes.dat similarity index 100% rename from test/test_inputs/tracker/passtimes.dat rename to IFTPipeline.jl/test/test_inputs/tracker/passtimes.dat diff --git a/test/test_inputs/tracker/tracker-params.toml b/IFTPipeline.jl/test/test_inputs/tracker/tracker-params.toml similarity index 100% rename from test/test_inputs/tracker/tracker-params.toml rename to IFTPipeline.jl/test/test_inputs/tracker/tracker-params.toml diff --git a/test/test_inputs/tracker/tracker_test_data.dat b/IFTPipeline.jl/test/test_inputs/tracker/tracker_test_data.dat similarity index 100% rename from test/test_inputs/tracker/tracker_test_data.dat rename to IFTPipeline.jl/test/test_inputs/tracker/tracker_test_data.dat diff --git a/PythonSetupForIFTPipeline.jl/.gitignore b/PythonSetupForIFTPipeline.jl/.gitignore new file mode 100644 index 00000000..2251642e --- /dev/null +++ b/PythonSetupForIFTPipeline.jl/.gitignore @@ -0,0 +1 @@ +Manifest.toml \ No newline at end of file diff --git a/PythonSetupForIFTPipeline.jl/README.md b/PythonSetupForIFTPipeline.jl/README.md index 7573ece9..27fccc72 100644 --- a/PythonSetupForIFTPipeline.jl/README.md +++ b/PythonSetupForIFTPipeline.jl/README.md @@ -26,3 +26,14 @@ julia --project="PythonSetupForIFTPipeline.jl" PythonSetupForIFTPipeline.jl/setu > rm -r ~/.julia/conda/ > ``` > ... and then retry. + +If you're working on a system where the home directory might not be reliable, +e.g. you're building a Docker container but you want to use it with Apptainer +which mounts things in different ways, you may want to change +the default location for the Conda installation. + +Export a different path in `CONDA_JL_HOME` before running setup: +```bash +export CONDA_JL_HOME="/opt/conda-env/" +julia --project="PythonSetupForIFTPipeline.jl" PythonSetupForIFTPipeline.jl/setup.jl +``` \ No newline at end of file diff --git a/PythonSetupForIFTPipeline.jl/setup.jl b/PythonSetupForIFTPipeline.jl/setup.jl index d32fee4d..6272fe72 100644 --- a/PythonSetupForIFTPipeline.jl/setup.jl +++ b/PythonSetupForIFTPipeline.jl/setup.jl @@ -1,15 +1,20 @@ -using Pkg -using Conda - # Instantiate all the dependencies +using Pkg Pkg.instantiate() # Initialize the Conda environment -ENV["PYTHON"]="" -Pkg.build("PyCall") +Pkg.build("Conda") # Add the dependencies to the Conda environment -Conda.runconda(Conda.Cmd(["env", "update", "-n", "base", "--file", joinpath(@__DIR__, "environment.yaml")])) +using Conda +Conda.runconda( + Conda.Cmd([ + "env", "update", "-n", "base", "--file", joinpath(@__DIR__, "environment.yaml") + ]), +) + +# Force PyCall to use the Conda version on Linux. +ENV["PYTHON"] = "" -# Rebuild PyCall with the new conda environment +# Build PyCall with the new conda environment Pkg.build("PyCall") diff --git a/README.md b/README.md index 6b2303c8..c998a48d 100644 --- a/README.md +++ b/README.md @@ -1,282 +1,17 @@ [![Build Status](https://github.com/WilhelmusLab/ice-floe-tracker-pipeline/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/WilhelmusLab/ice-floe-tracker-pipeline/actions/workflows/test.yml?query=branch%3Amain) [![Coverage](https://codecov.io/gh/WilhelmusLab/ice-floe-tracker-pipeline/branch/main/graph/badge.svg)](https://codecov.io/gh/WilhelmusLab/ice-floe-tracker-pipeline) # Ice Floe Tracker Pipeline -This repository contains the processing pipeline for IceFloeTracker.jl and ancillary scripts. +This repository contains the processing pipeline for IceFloeTracker.jl. -## SOIT Integration +The [workflow](./workflow/) directory includes instructions for running the workflow as a whole, in Cylc, and may be the only directory you need to interact with. -The [Satellite Overpass Identification Tool](https://zenodo.org/record/6475619#.ZBhat-zMJUe) is called to generate a list of satellite times for both Aqua and Terra in the area of interest. This program is written in Python and its dependencies are pulled from a Docker container at `docker://brownccv/icefloetracker-fetchdata:main`. +The other directories include utilities and are intended for developers: +- [PythonSetupForIFTPipeline.jl](./PythonSetupForIFTPipeline.jl/) – script to initialize a working Conda environment, in Julia +- [IFTPipeline.jl](./IFTPipeline.jl/) – command line interface to the ice floe tracker by Lopez ([IceFloeTracker.jl](https://github.com/WilhelmusLab/IceFloeTracker.jl)), in Julia +- [label-colorizer](./label-colorizer/) – command line tool to apply random colors to integer GEOTiff images, in Python +- [satellite-overpass-identification-tool](./satellite-overpass-identification-tool/) – command line tool to get satellite overpass times from the Aqua and Terra satellites, in Python +- [test](./test/) – command line scripts to test all the command line tool functionality -Register an account with [space-track.org](https://www.space-track.org/) to use SOIT. +The related repository [ebseg](https://github.com/WilhelmusLab/ebseg/) is also used, and provides an alternative preprocessing pipeline by Buckley. -To run SOIT manually : -1. Make sure Docker Desktop is running on your local machine. -2. Export SOIT username/password to environment variable. - - [ ] From your home directory `nano .bash_profile` - - [ ] add `export HISTCONTROL=ignoreboth` to the bottom of your .bash_profile - * this will ensure that your username/password are not stored in history - * when exporting the following environment variables, there __must__ be a space in front of each command - - [ ] ` export SPACEUSER=_@brown.edu` - - [ ] ` export SPACEPSWD=` -3. Update inputs and run the following: -```bash -docker run --env SPACEUSER --env SPACEPSWD --mount type=bind,source=,target=/tmp brownccv/icefloetracker-fetchdata:main \ -python3 /usr/local/bin/pass_time_cylc.py --startdate --enddate --csvoutpath /tmp --centroid_lat --centroid_lon --SPACEUSER $SPACEUSER --SPACEPSWD $SPACEPSWD -``` - * be sure to replace `source`, `startdate`, `enddate`, `centroid_lat`, and `centroid_lon` with your desired inputs - * csvoutpath must remain as `/tmp` to bind the Docker container output path with your desired local path - -**Note:** The `pass_time_cylc.py` script in this project can be adapted to include additional satellites available in the [space-track.org](https://www.space-track.org/) repository. If you have `numpy` and `skyfield` installed in a local `conda` environment, you can run `pass_time_cylc.py` from the directory where you installed the Ice Floe Tracker Pipeline: -```python3 workflow/scripts/pass_time_cylc.py --startdate --enddate --csvoutpath --centroid_x --centroid_y --SPACEUSER $SPACEUSER --SPACEPSWD $SPACEPSWD``` - -## Fetching data from NASA Worldview - -All the software dependencies to run `fetchdata.sh` are found in the Docker container at `docker://brownccv/icefloetracker-fetchdata:main`. - -To fetch data independently of other tasks: -1. Make sure Docker Desktop is running on your local machine. -2. Update inputs and run the following: -```bash -docker run --mount type=bind,source=,target=/tmp \ -brownccv/icefloetracker-fetchdata:main \ -/usr/local/bin/fetchdata.sh -o /tmp -s -e -c -b /ice-floe-tracker-pipeline` - - [ ] `conda env create -f ./config/ift-env.yaml` - - [ ] `conda activate ift-env` - -3. Make sure you have registered for an account with `space-track.org` and exported your SOIT credentials as an environment variable on Oscar as outlined in the [SOIT integration](#soit-integration) section. - -4. Prepare the runtime environment - - Cylc will use software dependencies inside a Singularity container to fetch images and satellite times from external APIs. - - [ ] It is a good idea to reset the Singularity cache dir to `scratch` as specified [here](https://docs.ccv.brown.edu/oscar/singularity-containers/building-images). Images take up a lot of space and `scratch` gets cleaned regularly. - - - [ ] first populate the `flow.cylc` file by running: - ```python - python workflow/scripts/flow_generator.py \ - --csvfile "./config/" \ - --template "flow_template_hpc.j2" \ - --template_dir "./config/cylc_hpc" \ - --crs "" \ - --minfloearea \ - --maxfloearea - ``` - (replacing `` with the name of your CSV file.) -Run `python workflow/scripts/flow_generator.py --help` for a list of options. - - - [ ] then, build the workflow, run it, and open the Terminal-based User Interface (TUI) to monitor the progress of each task. - ![TUI example](./tui-example.png) - - ```bash - cylc install -n ./config/cylc_hpc - cylc validate - cylc play - cylc tui - ``` - You can also get an image of the task dependency graph with `cylc graph `; you have to click the generated link to open it in VS Code. - ![Graph example](./graph-example.PNG) - - - [ ] If you need to change parameters and re-run a workflow, first do: - - ```bash - cylc stop --now - cylc clean - ``` - - [ ] Then, proceed to install, play, and open the TUI - -**Note:** Error logs are available for each task: - - ```bash - cat ./cylc-run///log/job/1//01/job.err - ``` - The entire `cylc-run` workflow generated by Cylc is also symlinked to `~/ice-floe-tracker-pipeline/workflow/cylc-run/`. - Julia logging is also available at `~/ice-floe-tracker-pipeline/workflow/report/` - - Failed tasks with retry automatically. If all retrys fail, there are likely too many clouds in the study area for the given dates. Try using the NASA Worldview web app to find better dates with fewer clouds. For large bounding boxes, you may need to increase the memory or cpus-per-task flags in the `cylc_hpc/flow.cylc` file for tasks that are failing. - -### Running the Cylc pipeline locally - -#### Prerequisites -__Julia:__ When running locally, make sure you have at least Julia 1.9.0 installed with the correct architecture for your local machine. (https://julialang.org/downloads/) - -__Docker Desktop:__ Also make sure Docker Desktop client is running in the background to use the Cylc pipeline locally. (https://www.docker.com/products/docker-desktop/) - -1. Build a virtual environment and install Cylc - - [ ] `cd /ice-floe-tracker-pipeline` - - [ ] `conda env create -f ./config/ift-env.yaml` - - [ ] `conda activate ift-env` - -**Note:** Depending on your existing Conda config, you may need to update your `.condarc` file to: `auto_activate_base: false` if you get errors running your first Cylc workflow. - -2. Make sure you have registered for an account with `space-track.org` and exported your SOIT credentials as an environment variable on your local computer as outlined in the [SOIT integration](#soit-integration) section. - -3. Install your workflow, run it, and monitor with the Terminal User Interface (TUI) - - - [ ] first populate the `flow.cylc` file by running: - ```python - python workflow/scripts/flow_generator.py \ - --csvfile "./config/" \ - --template "flow_template_local.j2" \ - --template_dir "./config/cylc_local" \ - --crs "" \ - --minfloearea \ - --maxfloearea - ``` - (replacing `` with the name of your CSV file.) - - Run `python workflow/scripts/flow_generator.py --help` for a list of options. - - - [ ] `cylc install -n ./config/cylc_local` - - [ ] `cylc graph #install graphviz locally` - - [ ] `cylc validate ` - - [ ] `cylc play ` - - [ ] `cylc tui ` - -The Terminal-based User Interface provides a simple way to watch the status of each task called in the `flow.cylc` workflow. Use arrow keys to investigate each task (see more [here](https://cylc.github.io/cylc-doc/latest/html/7-to-8/major-changes/ui.html#cylc-tui). -![TUI](tui-example.png)). - -If you need to change parameters and re-run a workflow, first do: - - ```bash - cylc stop --now - cylc clean - ``` - - [ ] Then, proceed to install, play, and open the TUI - - [ ] To rerun in one line: - ```bash - cylc stop --now && \ - cylc clean && \ - cylc install -n ./config/cylc_hpc && \ - cylc validate && \ - cylc play && \ - cylc tui - ``` - -**Note:** Error logs are available for each task: - - ```bash - cat ~/cylc-run///log/job/1//01/job.err - ``` - -### Tips for running the code locally for development - -When working locally, we have found VSCode to be a good interface for development. - -#### Cylc local development -When working locally and using the Cylc local pipeline, double check that the Docker client is running and clean the Docker cache to make sure you are using the latest images. -- [ ] delete any existing images from the Docker Dashboard -- [ ] from a terminal, run: -```bash -docker rm $(docker ps -aq) -``` -- [ ] from a terminal, run: -```bash -docker image prune -``` -#### Command-line interface (CLI) - local development -When running the CLI locally, make sure you have at least Julia 1.9.0 installed with the correct architecture for your local machine. (https://julialang.org/downloads/) - -- [ ] `cd /ice-floe-tracker-pipeline` - -Open a Julia REPL and build the package -- [ ] `julia` - -Enter Pkg mode and precompile -- [ ] `]` -- [ ] `activate .` -- [ ] `build` - -Use the backspace to go back to the Julia REPL and start running Julia code! - -__Note__ Use the help for wrapper scripts to learn about available options in each wrapper function -For example, from a bash prompt: -`julia --project=. ./workflow/scripts/ice-floe-tracker.jl extractfeatures --help` - -Here is the list of wrapper functions used in the CLI: -```bash -commands: - landmask Generate land mask images - preprocess Preprocess truecolor/falsecolor images - extractfeatures Extract ice floe features from segmented floe image - makeh5files Make HDF5 files from extracted floe features - track Pair ice floes in day k with ice floes in day k+1 - -optional arguments: - -h, --help show this help message and exit -``` -## Sample workflows to read in data stored in hdf5 files with Python - -Due to differences in multidimensional array representation between Julia and Python, special handling might be necessary. - -### Read floe properties table to a pandas dataframe - -```python -import h5py -import pandas as pd -from os.path import join - -dataloc = "results/hdf5-files" -filename = "20220914T1244.aqua.labeled_image.250m.h5" - -ift_data = h5py.File(join(dataloc, filename)) -df = pd.DataFrame( - data=ift_data["floe_properties"]["properties"][:].T, # note the transposition - columns=ift_data["floe_properties"]["column_names"][:].astype(str), -) - -df.head() - -``` - -### Simple visualization of a segmented image -```python -import numpy as np -import matplotlib.pyplot as plt - -ift_segmented = ift_data['floe_properties']['labeled_image'][:,:].T # note the transposition -ift_segmented = np.ma.masked_array(ift_segmented, mask=ift_segmented==0) - -fig, ax = plt.subplots() -ax.imshow(ift_segmented, cmap='prism') -``` +See the [Development Guide](./DEVELOPMENT.md) for instructions on setting up your environment. \ No newline at end of file diff --git a/config/README.md b/config/README.md deleted file mode 100644 index 3b8d78de..00000000 --- a/config/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Config Directory - -This directory contains any configuration used for the pipeline. diff --git a/config/cylc_hpc/flow_template_hpc.j2 b/config/cylc_hpc/flow_template_hpc.j2 deleted file mode 100644 index d7ea33a4..00000000 --- a/config/cylc_hpc/flow_template_hpc.j2 +++ /dev/null @@ -1,205 +0,0 @@ -# cycl template mytemplate.j2 -[scheduler] - allow implicit tasks = True -[task parameters] - # Update this variable to match the range of parameter sets - param_set = 0..{{ rows }} -[scheduling] - initial cycle point = PT1M #creates a datetime one minute from system time in UTC - [[graph]] - R1 = global_setup => mkpaths => pullfetchimage & pulljuliaimage => fetchdata & soit => landmask => preprocess => extractfeatures => tracking & exportH5 - -[runtime] - [[root]] - [[[environment]]] - # Update these variables with your run parameters - location = {{ location }} - startdate = {{ startdate }} - enddate = {{ enddate }} - crs = {{ crs }} # epsg3413 for polar stereographic: left_x@top_y@right_x@lower_y - # wgs84 for lat/lon: top_left_lat@top_left_lon@lower_right_lat@lower_right_lon - bounding_box = {{ bounding_box }} - centroid_lat = {{ center_lat }} - centroid_lon = {{ center_lon }} - minfloearea = {{ minfloearea }} - maxfloearea = {{ maxfloearea }} - project_dir = "~/ice-floe-tracker-pipeline" - - # Recommend using these default paths for output - t = ${CYLC_WORKFLOW_INITIAL_CYCLE_POINT} - julia_exec = "/usr/local/julia/bin/julia" - report_dir = $project_dir/"workflow/report" - results_dir = $project_dir/"results"/$t - fetchdata_dir = $project_dir/"resources"/$t - truecolor_dir = $fetchdata_dir/"truecolor" - falsecolor_dir = $fetchdata_dir/"falsecolor" - - [[global_setup]] - script = """ - mkdir -p ~/.cylc - mkdir -p ~/.cylc/flow - cp $project_dir/config/cylc_hpc/global.cylc ~/.cylc/flow - """ - - [[mkpaths]] - # Convert param location list to indexable bash array and make directories for each location - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - res=$results_dir/${location_array[$i]} - mkdir -p $fetchdata_dir/${location_array[$i]} - mkdir -p $res/soit - mkdir -p $res/landmasks - mkdir -p $res/preprocess - mkdir -p $res/tracker - mkdir -p $res/preprocess/hdf5-files - """ - - [[pullfetchimage]] - # Pull the latest Docker image containing software for fetching images from NASA Worldview API and satellite times from space-track.org - # Apptainer will check the image layers and only re-download if necessary - script = """ - apptainer pull --force $project_dir/fetchdata.simg docker://brownccv/icefloetracker-fetchdata:main - """ - platform = oscar - execution time limit = PT1H - [[[directives]]] - --mem = 12G - --cpus-per-task = 2 - - [[fetchdata]] - # Convert param lists to indexable bash arrays and fetch falsecolor, truecolor, and landmask for each location - script = """ - i=${CYLC_TASK_PARAM_param_set} - bbox_array=( $(echo $bounding_box | sed -e 's/,/ /g') ) - location_array=( $(echo $location | sed -e 's/,/ /g') ) - startdate_array=( $(echo $startdate | sed -e 's/,/ /g') ) - enddate_array=( $(echo $enddate | sed -e 's/,/ /g') ) - - apptainer exec --bind $fetchdata_dir/${location_array[$i]}:/tmp $project_dir/fetchdata.simg /usr/local/bin/fetchdata.sh -o /tmp -s ${startdate_array[$i]} -e ${enddate_array[$i]} -c $crs -b ${bbox_array[$i]} - """ - platform = oscar - execution time limit = PT1H - submission retry delays = 5*PT30S - execution retry delays = 5*PT6S - [[[directives]]] - --mem = 12G - --cpus-per-task = 4 - - [[soit]] - # Convert param lists to indexable bash arrays and fetch Aqua and Terra satellite times for each location - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - startdate_array=( $(echo $startdate | sed -e 's/,/ /g') ) - enddate_array=( $(echo $enddate | sed -e 's/,/ /g') ) - centroid_lat_array=( $(echo $centroid_lat| sed -e 's/,/ /g') ) - centroid_lon_array=( $(echo $centroid_lon | sed -e 's/,/ /g') ) - - apptainer exec --bind $results_dir/${location_array[$i]}/soit:/tmp $project_dir/fetchdata.simg python3 /usr/local/bin/pass_time_cylc.py --startdate ${startdate_array[$i]} --enddate ${enddate_array[$i]} --csvoutpath /tmp --centroid-lat ${centroid_lat_array[$i]} --centroid-lon ${centroid_lon_array[$i]} --SPACEUSER $SPACEUSER --SPACEPSWD $SPACEPSWD - """ - platform = oscar - execution time limit = PT1H - submission retry delays = 5*PT30S - execution retry delays = 5*PT6S - [[[directives]]] - --mem = 12G - --cpus-per-task = 4 - - [[pulljuliaimage]] - # Pull the latest Docker image containing Julia IceFloeTracker software - # apptainer will check the image layers and only re-download if necessary - script = """ - apptainer pull --force $project_dir/icefloetracker-julia.simg docker://brownccv/icefloetracker-julia:main - """ - platform = oscar - execution time limit = PT1H - submission retry delays = 2*PT30S - execution retry delays = 2*PT6S - [[[directives]]] - --mem = 32G - --cpus-per-task = 2 - - [[landmask]] - # Generate land masks in Julia from raw landmask file - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - - apptainer exec --env JULIA_DEPOT_PATH=$HOME/.julia:/opt/julia --bind $results_dir/${location_array[$i]}/landmasks:/tmp,$report_dir:/usr/local/report $project_dir/icefloetracker-julia.simg $julia_exec -t auto /usr/local/bin/ice-floe-tracker.jl landmask $fetchdata_dir/${location_array[$i]} /tmp - """ - platform = oscar - execution time limit = PT1H - submission retry delays = 2*PT30S - execution retry delays = PT6S - [[[directives]]] - --mem-per-cpu = 4G - --cpus-per-task = 8 - - [[preprocess]] - # Preprocess the satellite imagery and convert to binary - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - - apptainer exec --env JULIA_DEPOT_PATH=$HOME/.julia:/opt/julia --bind $results_dir/${location_array[$i]}/preprocess:/tmp,$report_dir:/usr/local/report $project_dir/icefloetracker-julia.simg $julia_exec -t auto /usr/local/bin/ice-floe-tracker.jl preprocess -t $fetchdata_dir/${location_array[$i]}/truecolor -r $fetchdata_dir/${location_array[$i]}/falsecolor -l $results_dir/${location_array[$i]}/landmasks -p $results_dir/${location_array[$i]}/soit -o /tmp - """ - platform = oscar - execution time limit = PT1H - submission retry delays = 2*PT30S - execution retry delays = PT6S - [[[directives]]] - --mem-per-cpu = 12G - --cpus-per-task = 20 - - [[extractfeatures]] - # Identify floes and extract floe metrics from binary processed images - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - preprocess_dir=$results_dir/${location_array[$i]}/preprocess - - apptainer exec --env JULIA_DEPOT_PATH=$HOME/.julia:/opt/julia --bind $preprocess_dir:/tmp,$report_dir:/usr/local/report $project_dir/icefloetracker-julia.simg $julia_exec -t auto /usr/local/bin/ice-floe-tracker.jl extractfeatures -i $preprocess_dir -o /tmp --minarea $minfloearea --maxarea $maxfloearea - """ - platform = oscar - execution time limit = PT1H - submission retry delays = 2*PT30S - execution retry delays = PT6S - [[[directives]]] - --mem = 18G - --cpus-per-task = 2 - - [[tracking]] - # Pair and track identified floes across days - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - preprocess_dir=$results_dir/${location_array[$i]}/preprocess - sample_img=$fetchdata_dir/${location_array[$i]}/truecolor/$(ls $fetchdata_dir/${location_array[$i]}/truecolor | head -1) - - apptainer exec --env JULIA_DEPOT_PATH=$HOME/.julia:/opt/julia --bind $results_dir/${location_array[$i]}/"tracker":/tmp,$report_dir:/usr/local/report $project_dir/icefloetracker-julia.simg $julia_exec -t auto /usr/local/bin/ice-floe-tracker.jl track --imgs $preprocess_dir --props $preprocess_dir --passtimes $preprocess_dir --latlon $sample_img --output /tmp - """ - platform = oscar - execution time limit = PT1H - submission retry delays = 2*PT30S - execution retry delays = PT6S - [[[directives]]] - --mem = 20G - --cpus-per-task = 1 - - [[exportH5]] - # Package intermediate and final outputs into HDF5 files - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - sample_img=$fetchdata_dir/${location_array[$i]}/truecolor/$(ls $fetchdata_dir/${location_array[$i]}/truecolor | head -1) - - apptainer exec --env JULIA_DEPOT_PATH=$HOME/.julia:/opt/julia --bind $results_dir/${location_array[$i]}/preprocess:/tmp,$report_dir:/usr/local/report $project_dir/icefloetracker-julia.simg $julia_exec -t auto /usr/local/bin/ice-floe-tracker.jl makeh5files --pathtosampleimg $sample_img --resdir /tmp - """ - platform = oscar - execution time limit = PT1H - submission retry delays = 2*PT30S - execution retry delays = 2*PT6S - [[[directives]]] - --mem = 18G - --cpus-per-task = 2 diff --git a/config/cylc_hpc/global.cylc b/config/cylc_hpc/global.cylc deleted file mode 100644 index 6453fce4..00000000 --- a/config/cylc_hpc/global.cylc +++ /dev/null @@ -1,11 +0,0 @@ -[platforms] - [[oscar]] - hosts = localhost - install target = localhost - job runner = slurm - retrieve job logs = True - -[install] - [[symlink dirs]] - [[[localhost]]] - run = ~/ice-floe-tracker-pipeline/workflow \ No newline at end of file diff --git a/config/cylc_local/flow_template_local.j2 b/config/cylc_local/flow_template_local.j2 deleted file mode 100644 index 5db87e55..00000000 --- a/config/cylc_local/flow_template_local.j2 +++ /dev/null @@ -1,130 +0,0 @@ -[scheduler] - allow implicit tasks = True -[task parameters] - # Update this variable to match the range of parameter sets - param_set = 0..{{ rows }} -[scheduling] - initial cycle point = PT1M #creates a datetime one minute from system time in UTC - [[graph]] - R1 = mkpaths => fetchdata & soit => landmask => preprocess => extractfeatures => tracking & exportH5 - -[runtime] - [[root]] - [[[environment]]] - # Update these variables with your run parameters - location = {{ location }} - startdate = {{ startdate }} - enddate = {{ enddate }} - crs = {{ crs }} # epsg3413 for polar stereographic: left_x@top_y@right_x@lower_y - # wgs84 for lat/lon: top_left_lat@top_left_lon@lower_right_lat@lower_right_lon - bounding_box = {{ bounding_box }} - centroid_x = {{ centroid_x }} - centroid_y = {{ centroid_y }} - minfloearea = {{ minfloearea }} - maxfloearea = {{ maxfloearea }} - project_dir="/Users/tdivoll/Projects/Wilhelmus/ice-floe-tracker-pipeline" #the location where you cloned the pipeline repo - - # Recommend using these default paths for output - t = ${CYLC_WORKFLOW_INITIAL_CYCLE_POINT} - julia_exec = "/usr/local/julia/bin/julia" - report_dir = $project_dir/"workflow/report" - results_dir = $project_dir/"results"/$t - fetchdata_dir = $project_dir/"resources"/$t - truecolor_dir = $fetchdata_dir/"truecolor" - falsecolor_dir = $fetchdata_dir/"falsecolor" - - [[mkpaths]] - # Convert param location list to indexable bash array and make directories for each location - # sed is used to tokenize the list of comma-separated inputs - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - res=$results_dir/${location_array[$i]} - mkdir -p $fetchdata_dir/${location_array[$i]} - mkdir -p $res/soit - mkdir -p $res/landmasks - mkdir -p $res/preprocess - mkdir -p $res/tracker - mkdir -p $res/preprocess/hdf5-files - """ - - [[fetchdata]] - # Convert param lists to indexable bash arrays and fetch falsecolor, truecolor, and landmask for each location - script = """ - i=${CYLC_TASK_PARAM_param_set} - bbox_array=( $(echo $bounding_box | sed -e 's/,/ /g') ) - location_array=( $(echo $location | sed -e 's/,/ /g') ) - startdate_array=( $(echo $startdate | sed -e 's/,/ /g') ) - enddate_array=( $(echo $enddate | sed -e 's/,/ /g') ) - - docker run --mount type=bind,source=$fetchdata_dir/${location_array[$i]},target=/tmp --mount type=bind,source=$report_dir,target=/usr/local/bin/../report brownccv/icefloetracker-fetchdata:main /usr/local/bin/fetchdata.sh -o /tmp -s ${startdate_array[$i]} -e ${enddate_array[$i]} -c $crs -b ${bbox_array[$i]} - """ - [[soit]] - # Convert param lists to indexable bash arrays and fetch Aqua and Terra satellite times for each location - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - startdate_array=( $(echo $startdate | sed -e 's/,/ /g') ) - enddate_array=( $(echo $enddate | sed -e 's/,/ /g') ) - centroid_x_array=( $(echo $centroid_x| sed -e 's/,/ /g') ) - centroid_y_array=( $(echo $centroid_y | sed -e 's/,/ /g') ) - - docker run --env SPACEUSER --env SPACEPSWD --mount type=bind,source=$results_dir/${location_array[$i]}/soit,target=/tmp --mount type=bind,source=$report_dir,target=/usr/local/bin/../report brownccv/icefloetracker-fetchdata:main python3 /usr/local/bin/pass_time_cylc.py --startdate ${startdate_array[$i]} --enddate ${enddate_array[$i]} --csvoutpath /tmp --centroid-lon ${centroid_x_array[$i]} --centroid-lat ${centroid_y_array[$i]} --SPACEUSER $SPACEUSER --SPACEPSWD $SPACEPSWD - """ - - [[landmask]] - # Generate land masks in Julia from raw landmask file - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - - docker run --mount type=bind,source=$results_dir/${location_array[$i]}/landmasks,target=/tmp \ - --mount type=bind,source=$report_dir,target=/usr/local/bin/../report \ - brownccv/icefloetracker-julia:main $julia_exec -t auto /usr/local/bin/ice-floe-tracker.jl landmask $fetchdata_dir/${location_array[$i]} /tmp - """ - - [[preprocess]] - # Preprocess the satellite imagery and convert to binary - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - - docker run --mount type=bind,source=$results_dir/${location_array[$i]}/preprocess,target=/tmp \ - --mount -type=bind,source=$report_dir,target=/usr/local/bin/../report \ - brownccv/icefloetracker-julia:main $julia_exec -t auto /usr/local/bin/ice-floe-tracker.jl preprocess -t $fetchdata_dir/${location_array[$i]}/truecolor -r $fetchdata_dir/${location_array[$i]}/falsecolor -l $results_dir/${location_array[$i]}/landmasks -p $results_dir/${location_array[$i]}/soit -o /tmp - """ - - [[extractfeatures]] - # Identify floes and extract floe metrics from binary processed images - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - preprocess_dir=$results_dir/${location_array[$i]}/preprocess - - docker run --mount type=bind,source=$preprocess_dir,target=/tmp \ - --mount type=bind,source=$report_dir,target=/usr/local/bin/../report \ - brownccv/icefloetracker-julia:main $julia_exec -t auto /usr/local/bin/ice-floe-tracker.jl extractfeatures -i $preprocess_dir -o /tmp --minarea $minfloearea --maxarea $maxfloearea - """ - - [[tracking]] - # Pair and track identified floes across days - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - preprocess_dir=$results_dir/${location_array[$i]}/preprocess - - docker run --mount type=bind,source=$results_dir/${location_array[$i]}/tracker,target=/tmp \ - --mount type=bind,source=$report_dir,target=/usr/local/bin/../report \ - brownccv/icefloetracker-julia:main $julia_exec -t auto /usr/local/bin/ice-floe-tracker.jl track --imgs $preprocess_dir --props $preprocess_dir --deltat $preprocess_dir --output /tmp - """ - - [[exportH5]] - # Package intermediate and final outputs into HDF5 files - script = """ - i=${CYLC_TASK_PARAM_param_set} - location_array=( $(echo $location | sed -e 's/,/ /g') ) - - docker run --mount type=bind,source=$results_dir/${location_array[$i]}/preprocess,target=/tmp,$report_dir:/usr/local/bin/../report \ - brownccv/icefloetracker-julia:main $julia_exec -t auto /usr/local/bin/ice-floe-tracker.jl makeh5files --pathtosampleimg $fetchdata_dir/${location_array[$i]}/truecolor/$(ls $fetchdata_dir/${location_array[$i]}/truecolor | head -1) --resdir /tmp - """ diff --git a/config/ift-env.yaml b/config/ift-env.yaml deleted file mode 100644 index 6f766ad4..00000000 --- a/config/ift-env.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: ift-env - -channels: - - conda-forge - -dependencies: - - cylc-flow=8.2.1 diff --git a/config/sample_site_locations.csv b/config/sample_site_locations.csv deleted file mode 100755 index 96e12058..00000000 --- a/config/sample_site_locations.csv +++ /dev/null @@ -1,3 +0,0 @@ -location,center_lat,center_lon,top_left_lat,top_left_lon,lower_right_lat,lower_right_lon,left_x,right_x,lower_y,top_y,startdate,enddate -beaufort_sea,75,-135,67.22298,-152.46426,79.32881,-94.68433,-2383879.497,-883879.4975,-750000,750000,2020-09-05,2020-09-08 -hudson_bay,60,-83,59.65687,-101.24295,57.54266,-66.04186,-2795941.755,-1295941.755,-3368686.029,-1868686.029,2020-09-06,2020-09-09 diff --git a/label-colorizer/.gitignore b/label-colorizer/.gitignore new file mode 100644 index 00000000..efa407c3 --- /dev/null +++ b/label-colorizer/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/label-colorizer/label_colorizer.py b/label-colorizer/label_colorizer.py new file mode 100644 index 00000000..3fbdb06f --- /dev/null +++ b/label-colorizer/label_colorizer.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3.11 +"""Utility to assign random colors to values in a grayscale integer array saved as a GeoTIFF.""" + +import typer +import pathlib +import rasterio +import numpy as np +from rasterio.enums import ColorInterp +import colorsys +import tqdm + +app = typer.Typer() + + +@app.command() +def main( + input_path: pathlib.Path, + output_path: pathlib.Path, + mask_value: int = 0, + seed: int = 42, + randomize: bool = True, +): + with rasterio.open(input_path) as in_dataset: + assert in_dataset.count == 1, "There should be only one band in the dataset" + band1 = in_dataset.read(1) + + max_value = int(band1.max()) + all_values = range(1, max_value + 1) + + rng = np.random.default_rng(seed) + if randomize: + hsv_tuples = {i: (rng.uniform(), 1, 1) for i in all_values} + else: + hsv_tuples = {i: (i * 1.0 / max_value, 1, 1) for i in all_values} + rgba_values = {i: (*colorsys.hsv_to_rgb(*hsv_tuples[i]), 1) for i in all_values} + rgba_values[mask_value] = (0, 0, 0, 0) + + red = np.zeros(band1.shape) + green = np.zeros(band1.shape) + blue = np.zeros(band1.shape) + alpha = np.zeros(band1.shape) + + for (x, y), value in tqdm.tqdm(np.ndenumerate(band1), total=np.prod(band1.shape)): + rgba = rgba_values[value] + red[x, y] = int(255 * rgba[0]) + green[x, y] = int(255 * rgba[1]) + blue[x, y] = int(255 * rgba[2]) + alpha[x, y] = int(255 * rgba[3]) + + new_profile = in_dataset.profile + new_profile.update( + nodata=0, + dtype="uint8", + count=4, + colorinterp=[ + ColorInterp.red, + ColorInterp.green, + ColorInterp.blue, + ColorInterp.alpha, + ], + ) + + with rasterio.open(output_path, mode="w", **new_profile) as out_dataset: + out_dataset.write(red, 1) + out_dataset.write(green, 2) + out_dataset.write(blue, 3) + out_dataset.write(alpha, 4) + + +if __name__ == "__main__": + app() diff --git a/label-colorizer/pyproject.toml b/label-colorizer/pyproject.toml new file mode 100644 index 00000000..9b15d94d --- /dev/null +++ b/label-colorizer/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "label-colorizer" +version = "0.1.0" +description = "Utility to generate more easily human readable versions of label images" +requires-python = ">=3.10" +authors = [ + { name = "John G. Holland", email = "john_holland1@brown.edu" }, +] + + +classifiers = [ # TODO: update these + "Development Status :: 3 - Alpha", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering :: GIS", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", +] + +dependencies = [ + "rasterio>=1.3.0", + "numpy >=1.26, <2.0.0", + "typer", + "tqdm" +] + +[project.scripts] +colorize = "label_colorizer:app" + + +[build-system] +requires = ["setuptools>=43.0.0", "wheel","hatchling"] +build-backend = "hatchling.build" \ No newline at end of file diff --git a/resources/README.md b/resources/README.md deleted file mode 100644 index 5e96496e..00000000 --- a/resources/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Resources Directory - -This directory contains any retrieved resources used in the pipeline. This includes the initial satellite images and land mask. \ No newline at end of file diff --git a/results/README.md b/results/README.md deleted file mode 100644 index 795cd914..00000000 --- a/results/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Results Directory - -This directory contains any obtained results by the pipeline. This can be final or intermediate results. diff --git a/satellite-overpass-identification-tool/Dockerfile b/satellite-overpass-identification-tool/Dockerfile new file mode 100644 index 00000000..1d3eeea7 --- /dev/null +++ b/satellite-overpass-identification-tool/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.11-bullseye +ENV TERM=xterm +RUN python3 -m pip install pipx +COPY . /opt/satellite-overpass-identification-tool/. +RUN pipx install --global /opt/satellite-overpass-identification-tool +ENTRYPOINT [ "soit" ] diff --git a/satellite-overpass-identification-tool/README.md b/satellite-overpass-identification-tool/README.md new file mode 100644 index 00000000..8ff20ea8 --- /dev/null +++ b/satellite-overpass-identification-tool/README.md @@ -0,0 +1,21 @@ +# Satellite Overpass Identification Tool + +The [Satellite Overpass Identification Tool](https://zenodo.org/record/6475619#.ZBhat-zMJUe) is called to generate a list of satellite times for both Aqua and Terra in the area of interest. + +## Run the code + +You can run the local version of the code from this directory by calling +```bash +pipx run . soit +``` + +You can run the code anywhere by calling: +```bash +# TODO: Remove branch specifier once merged +pipx run --spec "git+https://github.com/wilhelmuslab/ice-floe-tracker-pipeline@jghrefactor/C-update-soit-to-use-pipx#egg=satellite-overpass-identification-tool&subdirectory=satellite-overpass-identification-tool" soit +``` + +You can run the Docker image by calling: +```bash +docker run -it brownccv/icefloetracker-soit +``` diff --git a/satellite-overpass-identification-tool/pyproject.toml b/satellite-overpass-identification-tool/pyproject.toml new file mode 100644 index 00000000..ac0e756f --- /dev/null +++ b/satellite-overpass-identification-tool/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name="satellite-overpass-identification-tool" +version = "2024.11.4" +authors = [ + {name="Simon Hatcher"}, + {name="Timothy Divoll", email = "timothy_divoll@brown.edu"} +] +requires-python = ">= 3.10" +dependencies = [ + "requests==2.31.0", + "skyfield==1.45.0", + "numpy==1.26", +] + +[project.scripts] +soit="satellite_overpass_identification_tool:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" \ No newline at end of file diff --git a/workflow/scripts/pass_time_cylc.py b/satellite-overpass-identification-tool/satellite_overpass_identification_tool.py similarity index 81% rename from workflow/scripts/pass_time_cylc.py rename to satellite-overpass-identification-tool/satellite_overpass_identification_tool.py index cc746610..1eac3c55 100644 --- a/workflow/scripts/pass_time_cylc.py +++ b/satellite-overpass-identification-tool/satellite_overpass_identification_tool.py @@ -25,6 +25,7 @@ import csv import math import argparse +import pathlib # URLs for space track login. uriBase = "https://www.space-track.org" @@ -68,7 +69,7 @@ def get_passtimes(start_date, end_date, csvoutpath, lat, lon, SPACEUSER, SPACEPS rows = [] # Loop through each day until the end date of interest is reached. - while not np.array_equiv(tomorrow, end_date): + while not np.array_equiv(today, end_date): # Get UTC time values of the start of today and the start of tomorrow. # Passes between these times are considered. t0 = to_utc(today) @@ -90,13 +91,66 @@ def get_passtimes(start_date, end_date, csvoutpath, lat, lon, SPACEUSER, SPACEPS today = getNextDay(today) tomorrow = getNextDay(today) - csvwrite(start_date, end_date, lat, lon, rows, csvoutpath) + fields_, rows_ = convert_fields_mdy_folded_to_iso8601_unfolded(rows) + csvwrite(start_date, end_date, lat, lon, rows_, csvoutpath, fields=fields_) + +def convert_fields_mdy_folded_to_iso8601_unfolded(rows): + """Convert a row from [MM-DD-YYYY, UTC time (aqua), UTC time (terra)] to [YYYY-MM-DD, Satellite, ISO8601 datetime] format. + + Examples: + >>> convert_fields_mdy_folded_to_iso8601_unfolded([("03-31-2013", "11:50:20", "14:45:05"),]) + ... # doctest: +NORMALIZE_WHITESPACE + (['date', 'satellite', 'overpass time'], + [['2013-03-31', 'aqua', '2013-03-31T11:50:20Z'], + ['2013-03-31', 'terra', '2013-03-31T14:45:05Z']]) + + >>> convert_fields_mdy_folded_to_iso8601_unfolded([("12-01-2609", "23:59:01", "00:00:00"),]) + ... # doctest: +NORMALIZE_WHITESPACE + (['date', 'satellite', 'overpass time'], + [['2609-12-01', 'aqua', '2609-12-01T23:59:01Z'], + ['2609-12-01', 'terra', '2609-12-01T00:00:00Z']]) + + + >>> convert_fields_mdy_folded_to_iso8601_unfolded([ + ... ("03-31-2013", "11:50:20", "14:45:05"), + ... ("04-01-2013", "11:52:20", "14:43:05"), + ... ]) + ... # doctest: +NORMALIZE_WHITESPACE + (['date', 'satellite', 'overpass time'], + [['2013-03-31', 'aqua', '2013-03-31T11:50:20Z'], + ['2013-03-31', 'terra', '2013-03-31T14:45:05Z'], + ['2013-04-01', 'aqua', '2013-04-01T11:52:20Z'], + ['2013-04-01', 'terra', '2013-04-01T14:43:05Z']]) + + + """ + new_fields = ["date", "satellite", "overpass time"] + new_rows = [] + for row in rows: + date_mm_dd_yyyy, aqua_time, terra_time = row + m, d, y = map(int, date_mm_dd_yyyy.split("-")) + date_yyyy_mm_dd = datetime.date(y, m, d) + + new_rows.append([f"{date_yyyy_mm_dd}", "aqua", f"{date_yyyy_mm_dd}T{aqua_time}Z"]) + new_rows.append([f"{date_yyyy_mm_dd}", "terra", f"{date_yyyy_mm_dd}T{terra_time}Z"]) + + return new_fields, new_rows # Write CSV of all pass information. -def csvwrite(startdate, enddate, lat, lon, rows, outpath): - fields = ["Date", "Aqua pass time", "Terra pass time"] - filename = f"{outpath}/passtimes_lat{lat}_lon{lon}_{''.join(startdate)}_{''.join(enddate)}.csv" +def csvwrite(startdate, enddate, lat, lon, rows, outpath, fields=["Date", "Aqua pass time", "Terra pass time"]): + + outpath_ = pathlib.Path(outpath) + + if outpath_.is_dir(): + csv_name = f"passtimes_lat{lat}_lon{lon}_{''.join(startdate)}_{''.join(enddate)}.csv" + filename = outpath_ / pathlib.Path(csv_name) + elif outpath_.suffix == ".csv": + filename = outpath_ + else: + msg = "Output path neither a directory nor a .csv file: %s" % outpath + raise IOError(msg) + with open(filename, "w", newline="") as csvfile: csvwriter = csv.writer(csvfile) csvwriter.writerow(fields) @@ -367,7 +421,7 @@ def main(): parser.add_argument( "--csvoutpath", type=str, - help="Path to output CSV file", + help="Path to output CSV file, or a directory, where the output should be written", ) args = parser.parse_args() diff --git a/src/cli-config.jl b/src/cli-config.jl deleted file mode 100644 index 0470771d..00000000 --- a/src/cli-config.jl +++ /dev/null @@ -1,220 +0,0 @@ -function mkclipreprocess!(settings) - @add_arg_table! settings["preprocess"] begin - "--truedir", "-t" - help = "Truecolor image directory" - required = true - - "--fcdir", "-r" - help = "Falsecolor image directory" - required = true - - "--lmdir", "-l" - help = "Land mask image directory" - required = true - - "--passtimesdir", "-p" - help = "Pass times directory" - required = true - - "--output", "-o" - help = "Output directory" - required = true - end - return nothing -end - -function mkcliextract!(settings) - @add_arg_table! settings["extractfeatures"] begin - "--input", "-i" - help = "Input image directory" - required = true - - "--output", "-o" - help = "Output image directory" - required = true - - "--minarea" - help = "Minimum area (in pixels) of ice floes to extract" - default = "350" - - "--maxarea" - help = "Maximum area (in pixels) of ice floes to extract" - default = "90000" - - "--features", "-f" - help = """Features to extract. Format: "feature1 feature2". For an extensive list of extractable features see https://scikit-image.org/docs/stable/api/skimage.measure.html#skimage.measure.regionprops:~:text=The%20following%20properties%20can%20be%20accessed%20as%20attributes%20or%20keys""" - default = "centroid area major_axis_length minor_axis_length convex_area bbox orientation perimeter" - end - return nothing -end - -function mkclimakeh5!(settings) - @add_arg_table! settings["makeh5files"] begin - "--pathtosampleimg", "-p" - help = "Path to a sample image with coordinate reference system (CRS) and latitude and longitude coordinates of image pixels" - arg_type = String - - "--resdir", "-r" - help = "Path to the directory containing serialized results of the IceFloeTracker pipeline" - arg_type = String - end - return nothing -end - -""" - mkclitrack!(settings) - -Set up command line interface for the `track` command. -""" -function mkclitrack!(settings) - add_arg_group!(settings["track"], "arguments") - @add_arg_table! settings["track"] begin - "--imgs" - help = "Path to object with segmented images" - required = true - - "--props" - help = "Path to object with extracted features" - required = true - - "--passtimes" - help = "Path to object with satellite pass times" - required = true - - "--latlon" - help = "Path to geotiff image with latitude/longitude data" - required = true - - "--output", "-o" - help = "Output directory" - required = true - end - - add_arg_group!(settings["track"], "optional arguments") - @add_arg_table! settings["track"] begin - "--params" - help = "Path to TOML file with algorithm parameters" - - "--area" - help = "Area thresholds to use for pairing floes" - arg_type = Int64 - default = 1200 - - "--dist" - help = "Distance threholds to use for pairing floes" - default = "15 30 120" - - "--dt-thresh" - help = "Time thresholds to use for pairing floes" - default = "30 100 1300" - - "--Sarearatio" - help = "Area ratio threshold for small floes" - arg_type = Float64 - default = 0.18 - - "--Smajaxisratio" - help = "Major axis ratio threshold for small floes" - arg_type = Float64 - default = 0.1 - - "--Sminaxisratio" - help = "Minor axis ratio threshold for small floes" - arg_type = Float64 - default = 0.12 - - "--Sconvexarearatio" - help = "Convex area ratio threshold for small floes" - arg_type = Float64 - default = 0.14 - - "--Larearatio" - help = "Area ratio threshold for large floes" - arg_type = Float64 - default = 0.28 - - "--Lmajaxisratio" - help = "Major axis ratio threshold for large floes" - arg_type = Float64 - default = 0.1 - - "--Lminaxisratio" - help = "Minor axis ratio threshold for large floes" - arg_type = Float64 - default = 0.15 - - "--Lconvexarearatio" - help = "Convex area ratio threshold for large floes" - arg_type = Float64 - default = 0.14 - - # matchcorr computation - "--mxrot" - help = "Maximum rotation" - arg_type = Int64 - default = 10 - - "--psi" - help = "Minimum psi-s correlation" - arg_type = Float64 - default = 0.95 - - "--sz" - help = "Minimum side length of floe mask" - arg_type = Int64 - default = 16 - - "--comp" - help = "Size comparability" - arg_type = Float64 - default = 0.25 - - "--mm" - help = "Maximum registration mismatch" - arg_type = Float64 - default = 0.22 - - # Goodness of match - "--corr" - help = "Mininimun psi-s correlation" - arg_type = Float64 - default = 0.68 - - "--area2" - help = "Large floe area mismatch threshold" - arg_type = Float64 - default = 0.236 - - "--area3" - help = "Small floe area mismatch threshold" - arg_type = Float64 - default = 0.18 - end - return nothing -end - -function mkclilandmask!(settings) - args = [ - "input", - Dict(:help => "Input image directory", :required => true), - "output", - Dict(:help => "Output image directory", :required => true), - ] - add_arg_table!(settings["landmask"], args...) -end - -function mkcli!(settings, common_args) - d = Dict( - "landmask" => mkclilandmask!, - "preprocess" => mkclipreprocess!, - "extractfeatures" => mkcliextract!, - "makeh5files" => mkclimakeh5!, - "track" => mkclitrack! - ) - - for t in keys(d) - d[t](settings) # add arguments to settings - add_arg_table!(settings[t], common_args...) - end - return nothing -end diff --git a/src/track-docstring.jl b/src/track-docstring.jl deleted file mode 100644 index d64dbe8e..00000000 --- a/src/track-docstring.jl +++ /dev/null @@ -1,19 +0,0 @@ -""" -Pair floes in the floe library using an equivalent implementation as in the MATLAB script `final_2020.m` from https://github.com/WilhelmusLab/ice-floe-tracker/blob/main/existing_code/final_2020.m. - -# Arguments -- `indir`: path to directory containing the floe library (cropped floe masks for registration and correlation), extracted features from segmented images, and satellite overpass times. -- `condition_thresholds`: 3-tuple of thresholds (each a named tuple) for deciding whether to match floe `i` from day `k` to floe j from day `k+1`. -- `mc_thresholds`: thresholds for area mismatch and psi-s shape correlation and goodness of a match. - -See `IceFloeTracker.jl/notebooks/track-floes/track-floes.ipynb` for a sample workflow. - -Following are the default set of thresholds `condition_thresholds` used for floe matching: -- Condition 1: time elapsed `dt` from image `k` to image `k+1` and distance between floes centroids `dist`: `t1=(dt = (30, 100, 1300), dist=(15, 30, 120))` - -- Condition 2: area of floe i `area`, and the computed ratios for area, major axis, minor axis, and convex area of floes `i` and `j` in days `k` and `k+1`, respectively: `t2=(area=1200, arearatio=0.28, majaxisratio=0.10, minaxisratio=0.12, convexarearatio=0.14)` - -- Condition 3: as in the previous condition but set as follows: ` -t3=(area=1200, arearatio=0.18, majaxisratio=0.07, minaxisratio=0.08, convexarearatio=0.09)` - -""" \ No newline at end of file diff --git a/src/tracker.jl b/src/tracker.jl deleted file mode 100644 index 696d7360..00000000 --- a/src/tracker.jl +++ /dev/null @@ -1,83 +0,0 @@ -""" - track( - imgsdir::String, - propsdir::String, - passtimesdir::String, - paramsdir::String, - outdir::String, -) - -$(include("track-docstring.jl")) -""" -function track(; args...) - condition_thresholds, mc_thresholds = get_thresholds(; args...) - vals = NamedTuple{Tuple(keys(args))}(values(args)) # convert to NamedTuple - imgs = deserialize(joinpath(vals.imgs, "segmented_floes.jls")) - props = deserialize(joinpath(vals.props, "floe_props.jls")) - passtimes = deserialize(joinpath(vals.passtimes, "passtimes.jls")) - latlon = vals.latlon - labeledfloes = IceFloeTracker.pairfloes(imgs, props, passtimes, latlon, condition_thresholds, mc_thresholds) - serialize(joinpath(vals.output, "labeled_floes.jls"), labeledfloes) - return nothing -end - -function parse_params(params::AbstractString) - params = parsefile(params) - area = params["area"] - t1 = dict2nt(params["t1"]) - t2 = (area=area, dict2nt(params["t2"])...) - t3 = (area=area, dict2nt(params["t3"])...) - condition_thresholds = (t1=t1, t2=t2, t3=t3) - d = dict2nt(params["mc_thresholds"]) - mc_thresholds = mkmct(d) - return condition_thresholds, mc_thresholds -end - -function parse_params(; args...) - d = values(args) - condition_thresholds = ( - t1=(dt=parselistas(Int64, d.dt_thresh), dist=parselistas(Int64, d.dist)), - t2=( - area=d.area, - arearatio=d.Larearatio, - convexarearatio=d.Lconvexarearatio, - majaxisratio=d.Lmajaxisratio, - minaxisratio=d.Lminaxisratio, - ), - t3=( - area=d.area, - arearatio=d.Sarearatio, - convexarearatio=d.Sconvexarearatio, - majaxisratio=d.Smajaxisratio, - minaxisratio=d.Sminaxisratio, - ), - ) - mc_thresholds = mkmct(d) - return condition_thresholds, mc_thresholds -end - -function mkmct(d) - return ( - comp=(mxrot=d.mxrot, sz=d.sz, comp=d.comp, mm=d.mm, psi=d.psi), - goodness=(corr=d.corr, area2=d.area2, area3=d.area3), - ) -end - -function get_thresholds(; args...) - v = values(args) - if !isnothing(v.params) - condition_thresholds, mc_thresholds = parse_params(v.params) - else - condition_thresholds, mc_thresholds = parse_params(; args...) - end - return condition_thresholds, mc_thresholds -end - -""" - dict2nt(d) - -Convert a dictionary `d` to a NamedTuple. -""" -dict2nt(d) = NamedTuple((Symbol(key), value) for (key, value) in d) - -parselistas(T, x) = [parse(T, i) for i in split(x)] diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 00000000..875e9946 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +tmp.* \ No newline at end of file diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..f9ae8194 --- /dev/null +++ b/test/README.md @@ -0,0 +1,45 @@ +# CLI Test scripts + +Scripts to test local and dockerized version of the ice floe tracker CLI. + +To run the tests using the local version of the Ice Floe Tracker CLI, first: +```bash +source ./test-IFTPipeline.jl-cli.sh +``` + +Then run the appropriate function with the dataset you would like to test. Preprocessing: +```bash +preprocess_lopez input_data/ne-greenland.20220913.terra.250m/ +preprocess_buckley input_data/ne-greenland.20220913.terra.250m/ +``` + +For tracking (and preprocessing), you need to pass all the source data directories you'd like to process: +```bash +# `track` uses dependency injection to select the preprocessing pipeline +PREPROCESS=preprocess_lopez track input_data/ne-greenland.2022091{3,4}.terra.250m/ + +# `track_original` and `track_buckley` are wrappers around `track` +track_original input_data/ne-greenland.2022091{3,4}.terra.250m/ +track_buckley input_data/ne-greenland.2022091{3,4}.terra.250m/ +``` + +To run the tests using a dockerized version of the Ice Floe Tracker CLI, call: +```bash +IFT="docker run -v `pwd`:/app -w /app brownccv/icefloetracker-julia" ... +``` +... where `brownccv/icefloetracker-julia` can be replaced with any local or remote tagged version of the ice floe tracker image. + +For example: +```bash +IFT="docker run -v `pwd`:/app -w /app brownccv/icefloetracker-julia" track_buckley input_data/ne-greenland.2022091{3,4}.terra.250m/ +``` + +Or with Apptainer: +```bash +IFT="apptainer run brownccv/icefloetracker-julia:pr-198" preprocess_lopez input_data/ne-greenland.20220913.terra.250m/ +``` + +or +```bash +IFT="apptainer run `pwd`/../runtime/image/iftp-pr-198.simg" preprocess_lopez input_data/ne-greenland.20220913.terra.250m/ +``` diff --git a/test/input_data/ne-greenland.20220913.terra.250m/cloud.tiff b/test/input_data/ne-greenland.20220913.terra.250m/cloud.tiff new file mode 100644 index 00000000..da2d5a8e Binary files /dev/null and b/test/input_data/ne-greenland.20220913.terra.250m/cloud.tiff differ diff --git a/test/input_data/ne-greenland.20220913.terra.250m/falsecolor.tiff b/test/input_data/ne-greenland.20220913.terra.250m/falsecolor.tiff new file mode 100644 index 00000000..827eadb5 Binary files /dev/null and b/test/input_data/ne-greenland.20220913.terra.250m/falsecolor.tiff differ diff --git a/test/input_data/ne-greenland.20220913.terra.250m/landmask.tiff b/test/input_data/ne-greenland.20220913.terra.250m/landmask.tiff new file mode 100644 index 00000000..4ddbf601 Binary files /dev/null and b/test/input_data/ne-greenland.20220913.terra.250m/landmask.tiff differ diff --git a/test/input_data/ne-greenland.20220913.terra.250m/overpass.txt b/test/input_data/ne-greenland.20220913.terra.250m/overpass.txt new file mode 100644 index 00000000..fd09d846 --- /dev/null +++ b/test/input_data/ne-greenland.20220913.terra.250m/overpass.txt @@ -0,0 +1 @@ +2022-09-13T00:00:00 diff --git a/test/input_data/ne-greenland.20220913.terra.250m/truecolor.tiff b/test/input_data/ne-greenland.20220913.terra.250m/truecolor.tiff new file mode 100644 index 00000000..3c34a8ca Binary files /dev/null and b/test/input_data/ne-greenland.20220913.terra.250m/truecolor.tiff differ diff --git a/test/input_data/ne-greenland.20220914.aqua.250m/cloud.tiff b/test/input_data/ne-greenland.20220914.aqua.250m/cloud.tiff new file mode 100644 index 00000000..56bec7c7 Binary files /dev/null and b/test/input_data/ne-greenland.20220914.aqua.250m/cloud.tiff differ diff --git a/test/input_data/ne-greenland.20220914.aqua.250m/falsecolor.tiff b/test/input_data/ne-greenland.20220914.aqua.250m/falsecolor.tiff new file mode 100644 index 00000000..892bf3a1 Binary files /dev/null and b/test/input_data/ne-greenland.20220914.aqua.250m/falsecolor.tiff differ diff --git a/test/input_data/ne-greenland.20220914.aqua.250m/landmask.tiff b/test/input_data/ne-greenland.20220914.aqua.250m/landmask.tiff new file mode 100644 index 00000000..c148055f Binary files /dev/null and b/test/input_data/ne-greenland.20220914.aqua.250m/landmask.tiff differ diff --git a/test/input_data/ne-greenland.20220914.aqua.250m/overpass.txt b/test/input_data/ne-greenland.20220914.aqua.250m/overpass.txt new file mode 100644 index 00000000..d8a9c5e7 --- /dev/null +++ b/test/input_data/ne-greenland.20220914.aqua.250m/overpass.txt @@ -0,0 +1 @@ +2022-09-14T12:44:49 diff --git a/test/input_data/ne-greenland.20220914.aqua.250m/truecolor.tiff b/test/input_data/ne-greenland.20220914.aqua.250m/truecolor.tiff new file mode 100644 index 00000000..e67f09cf Binary files /dev/null and b/test/input_data/ne-greenland.20220914.aqua.250m/truecolor.tiff differ diff --git a/test/input_data/ne-greenland.20220914.terra.250m/cloud.tiff b/test/input_data/ne-greenland.20220914.terra.250m/cloud.tiff new file mode 100644 index 00000000..7c9d6c24 Binary files /dev/null and b/test/input_data/ne-greenland.20220914.terra.250m/cloud.tiff differ diff --git a/test/input_data/ne-greenland.20220914.terra.250m/falsecolor.tiff b/test/input_data/ne-greenland.20220914.terra.250m/falsecolor.tiff new file mode 100644 index 00000000..7df39bb8 Binary files /dev/null and b/test/input_data/ne-greenland.20220914.terra.250m/falsecolor.tiff differ diff --git a/test/input_data/ne-greenland.20220914.terra.250m/landmask.tiff b/test/input_data/ne-greenland.20220914.terra.250m/landmask.tiff new file mode 100644 index 00000000..4ddbf601 Binary files /dev/null and b/test/input_data/ne-greenland.20220914.terra.250m/landmask.tiff differ diff --git a/test/input_data/ne-greenland.20220914.terra.250m/overpass.txt b/test/input_data/ne-greenland.20220914.terra.250m/overpass.txt new file mode 100644 index 00000000..3138889f --- /dev/null +++ b/test/input_data/ne-greenland.20220914.terra.250m/overpass.txt @@ -0,0 +1 @@ +2022-09-14T00:00:00 diff --git a/test/input_data/ne-greenland.20220914.terra.250m/truecolor.tiff b/test/input_data/ne-greenland.20220914.terra.250m/truecolor.tiff new file mode 100644 index 00000000..a9892b9d Binary files /dev/null and b/test/input_data/ne-greenland.20220914.terra.250m/truecolor.tiff differ diff --git a/test/test-IFTPipeline.jl-cli.sh b/test/test-IFTPipeline.jl-cli.sh new file mode 100755 index 00000000..b34eacd8 --- /dev/null +++ b/test/test-IFTPipeline.jl-cli.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash + +# Set variables with the script names we're going to use. +# If these are set as environment variables outside, then the outside version will be used. +# This might be useful if you're trying to test a Docker container – +# you can replace IFT with the version from your docker container. +# The `:` at the start is deep bash magic. +# Refs: +# - https://stackoverflow.com/a/28085062/24937841 and +# - https://unix.stackexchange.com/a/31712 +: "${IFT:=julia --project=`pwd`/../IFTPipeline.jl `pwd`/../IFTPipeline.jl/src/cli.jl}" +: "${FSDPROC:=pipx run --spec git+https://github.com/wilhelmuslab/ebseg fsdproc --debug}" +: "${COLORIZE:=pipx run --spec git+https://github.com/wilhelmuslab/ice-floe-tracker-pipeline@merge-attempt-1#egg=label-colorizer&subdirectory=label-colorizer colorize }" + +# Set up debug messages +export JULIA_DEBUG="Main,IFTPipeline,IceFloeTracker" + +echo_CLI_tools () { + echo "IFT=${IFT}" + echo "FSDPROC=${FSDPROC}" + echo "COLORIZE=${COLORIZE}" +} + +initialize_test_directory () { + DATA_SOURCE=$1 + DATA_TARGET=$2 + + # Initialize data directory + : "${DATA_SOURCE:=./input_data}" + echo "DATA_SOURCE=${DATA_SOURCE}" + + TEMPDIR=$(mktemp -d -p .) + : "${DATA_TARGET:=$TEMPDIR}" + echo "DATA_TARGET=${DATA_TARGET}" + + mkdir -p ${DATA_TARGET}/ + + cp -r ${DATA_SOURCE}/* ${DATA_TARGET}/ +} + +preprocess_lopez () { + echo_CLI_tools + + DATA_SOURCE=$1 + DATA_TARGET=$2 + initialize_test_directory $1 $2 + + LANDMASK=${DATA_TARGET}/landmask.tiff + LANDMASK_NON_DILATED=${DATA_TARGET}/landmask.non-dilated.tiff + LANDMASK_DILATED=${DATA_TARGET}/landmask.dilated.tiff + TRUECOLOR=${DATA_TARGET}/truecolor.tiff + FALSECOLOR=${DATA_TARGET}/falsecolor.tiff + LABELED=${DATA_TARGET}/labeled.tiff + COLORIZED=${DATA_TARGET}/labeled.colorized.tiff + FLOEPROPERTIES=${DATA_TARGET}/labeled.props.csv + HDF5FILE=${DATA_TARGET}/results.h5 + OVERPASS=${DATA_TARGET}/overpass.txt + + ${IFT} landmask_single \ + -i ${LANDMASK} \ + -o ${LANDMASK_NON_DILATED} \ + -d ${LANDMASK_DILATED} + + ${IFT} preprocess_single \ + --truecolor ${TRUECOLOR} \ + --falsecolor ${FALSECOLOR} \ + --landmask ${LANDMASK_NON_DILATED} \ + --landmask-dilated ${LANDMASK_DILATED} \ + --output ${LABELED} + + ${IFT} extractfeatures_single \ + --input ${LABELED} \ + --output ${FLOEPROPERTIES} + + ${COLORIZE} ${LABELED} ${COLORIZED} + + ${IFT} makeh5files_single \ + --passtime `cat ${OVERPASS}` \ + --truecolor ${TRUECOLOR} \ + --falsecolor ${FALSECOLOR} \ + --labeled ${LABELED} \ + --props ${FLOEPROPERTIES} \ + --output ${HDF5FILE} +} + +preprocess_buckley () { + echo_CLI_tools + + DATA_SOURCE=$1 + DATA_TARGET=$2 + initialize_test_directory $1 $2 + + TRUECOLOR=${DATA_TARGET}/truecolor.tiff + FALSECOLOR=${DATA_TARGET}/falsecolor.tiff + CLOUD=${DATA_TARGET}/cloud.tiff + LANDMASK=${DATA_TARGET}/landmask.tiff + LABELED=${DATA_TARGET}/labeled.tiff + COLORIZED=${DATA_TARGET}/labeled.colorized.tiff + LABELEDDIR=${LABELED}.work/ + FLOEPROPERTIES=${DATA_TARGET}/labeled.props.csv + HDF5FILE=${DATA_TARGET}/results.h5 + OVERPASS=${DATA_TARGET}/overpass.txt + + ${FSDPROC} process ${TRUECOLOR} ${CLOUD} ${LANDMASK} ${LABELEDDIR} + cp ${LABELEDDIR}/final.tif ${LABELED} + cp ${LABELEDDIR}/props.csv ${FLOEPROPERTIES} + ${COLORIZE} ${LABELED} ${COLORIZED} + + ${IFT} makeh5files_single \ + --passtime `cat ${OVERPASS}` \ + --truecolor ${TRUECOLOR} \ + --falsecolor ${FALSECOLOR} \ + --labeled ${LABELED} \ + --props ${FLOEPROPERTIES} \ + --output ${HDF5FILE} +} + + +track () { + echo_CLI_tools + + DATA_SOURCES=$@ + + TEMPDIR=$(mktemp -d -p .) + : "${_DATA_TARGET:=$TEMPDIR}" + echo "_DATA_TARGET=${_DATA_TARGET}" + + _DATA_TARGET_SUBDIRS=() + + for source in ${DATA_SOURCES[@]} + do + _THIS_SUBDIR=${_DATA_TARGET}/$(basename $source)/ + _DATA_TARGET_SUBDIRS+=(${_THIS_SUBDIR}) + ${PREPROCESS} ${source} ${_THIS_SUBDIR} + done + + ${IFT} track_single \ + --imgs "${_DATA_TARGET_SUBDIRS[@]/%/labeled.tiff}" \ + --props "${_DATA_TARGET_SUBDIRS[@]/%/labeled.props.csv}" \ + --latlon "${_DATA_TARGET_SUBDIRS[1]/%/truecolor.tiff}" \ + --passtimes $(cat ${_DATA_TARGET_SUBDIRS[@]/%/overpass.txt} | tr '\n' ' ') \ + --output ${_DATA_TARGET}/paired.csv +} + + +track_original () { + PREPROCESS=preprocess_lopez track $@ +} + +track_buckley () { + PREPROCESS=preprocess_buckley track $@ +} diff --git a/workflow/.cylcignore b/workflow/.cylcignore new file mode 100644 index 00000000..737e26b0 --- /dev/null +++ b/workflow/.cylcignore @@ -0,0 +1 @@ +run/ \ No newline at end of file diff --git a/workflow/.gitignore b/workflow/.gitignore new file mode 100644 index 00000000..737e26b0 --- /dev/null +++ b/workflow/.gitignore @@ -0,0 +1 @@ +run/ \ No newline at end of file diff --git a/workflow/README.md b/workflow/README.md new file mode 100644 index 00000000..8b2cdc81 --- /dev/null +++ b/workflow/README.md @@ -0,0 +1,63 @@ +# Workflow + +This directory includes tools to run the ice floe tracker analysis on a batch of satellite images. + +## General Use + +### Prerequisites + +You will need the following installed on your computer: +- [`pipx`](https://pipx.pypa.io/stable/) +- [`cylc`](https://cylc.github.io/) + - the `cylc-rose` plugin + - [`global.cylc`](https://cylc.github.io/cylc-doc/stable/html/reference/config/global.html#global.cylc) file, making any modifications you might want. +- [`docker`](https://docs.docker.com/) + +You will also need a username and account on [space-track.org](https://space-track.org) + +### Running the pipeline + +Make a new configuration file with the region and time period you want to analyse. All the possible parameters are listed in [rose-suite.conf](./rose-suite.conf). You can see examples in [example](./example/). + +Add the following lines to the rose-suite.conf file with your space-track.org username and password: +``` +SPACEUSER="your_email_address@example.com" +SPACEPSWD="yourpassword" +``` + +Don't commit these changes to the repo. + + + +Run the pipeline by calling: +```bash +cylc vip . --set-file /path/to/your/configuration/file.conf -n your-analysis-run-name +``` + +Command line usage examples are shown in [example-cylc-calls.sh](./example-cylc-calls.sh). + +View the running commands by calling `cylc tui`. + +## Oscar + +Contact the Wilhelmus Lab members for access to the pipeline environment on Oscar, Brown University's High Performance Computing cluster. + +## Prerequisites + +`pipx`, `cylc` and `cylc-rose` are available in the `../runtime/venv` virtual environment. Load it using: + +```bash +module load python/3.11.0s-ixrhc3q +. ../runtime/venv/bin/activate +``` + +Update your `~/.cylc/flow/global.cylc` file to include the following lines to use the SLURM scheduler: +``` +[platforms] + [[localhost]] + job runner = slurm +``` + +This will ensure that jobs from `cylc` are scheduled using `Slurm`. + +Oscar uses Apptainer rather than Docker, so include the line `IFT_INSTALL="Apptainer"` or `IFT_INSTALL="ApptainerLocal"` in your configuration file. \ No newline at end of file diff --git a/workflow/example-cylc-calls.sh b/workflow/example-cylc-calls.sh new file mode 100755 index 00000000..29ee9ac7 --- /dev/null +++ b/workflow/example-cylc-calls.sh @@ -0,0 +1,20 @@ +# Copy oscar configuration to your home directory +mkdir -p ~/.cylc/flow && cp oscar.global.cylc ~/.cylc/flow/global.cylc + +cylc vip . --set-file example/iftpipeline-test-case.conf -n iftpipeline-test-case-lopez -s 'PREPROCESSING="Lopez"' +cylc vip . --set-file example/iftpipeline-test-case.conf -n iftpipeline-test-case-buckley -s 'PREPROCESSING="Buckley"' + +cylc vip . --set-file example/beaufort-sea-july.conf -n beaufort-sea-july-lopez -s 'PREPROCESSING="Lopez"' +cylc vip . --set-file example/beaufort-sea-july.conf -n beaufort-sea-july-buckley -s 'PREPROCESSING="Buckley"' + +cylc vip . --set-file example/beaufort-sea-march.conf -n beaufort-sea-march-lopez -s 'PREPROCESSING="Lopez"' +cylc vip . --set-file example/beaufort-sea-march.conf -n beaufort-sea-march-buckley -s 'PREPROCESSING="Buckley"' + +cylc vip . --set-file example/fram-strait-june.conf -n fram-strait-june-lopez -s 'PREPROCESSING="Lopez"' +cylc vip . --set-file example/fram-strait-june.conf -n fram-strait-june-buckley -s 'PREPROCESSING="Buckley"' + +cylc vip . --set-file example/fram-strait-april-may-2020.conf -n fram-strait-april-may-2020-lopez -s 'PREPROCESSING="Lopez"' +cylc vip . --set-file example/fram-strait-april-may-2020.conf -n fram-strait-april-may-2020-buckley -s 'PREPROCESSING="Buckley"' + +cylc vip . --set-file example/ne-greenland.conf -n ne-greenland-lopez -s 'PREPROCESSING="Lopez"' +cylc vip . --set-file example/ne-greenland.conf -n ne-greenland-buckley -s 'PREPROCESSING="Buckley"' \ No newline at end of file diff --git a/workflow/example/beaufort-sea-july.conf b/workflow/example/beaufort-sea-july.conf new file mode 100644 index 00000000..ff0306ac --- /dev/null +++ b/workflow/example/beaufort-sea-july.conf @@ -0,0 +1,9 @@ +START="2024-07-11" +END="2024-07-13" +SATELLITES="aqua", "terra" +LOCATION="beaufort-sea" +CRS="EPSG:3413" +BBOX="-1688750,-110000,-1283250,317750" +SCALE=250 +CENTROID_LAT=76.31228950602991 +CENTROID_LON=-139.00136296848805 \ No newline at end of file diff --git a/workflow/example/beaufort-sea-march.conf b/workflow/example/beaufort-sea-march.conf new file mode 100644 index 00000000..7fe643fa --- /dev/null +++ b/workflow/example/beaufort-sea-march.conf @@ -0,0 +1,9 @@ +START="2019-03-19" +END="2019-03-20" +SATELLITES="aqua", "terra" +LOCATION="beaufort-sea" +CRS="EPSG:3413" +BBOX="-2112500.0,-262500.0,-2012500.0,-162500.0" +SCALE=250 +CENTROID_LAT=71.02795656503653 +CENTROID_LON=-129.11755440990464 \ No newline at end of file diff --git a/workflow/example/fram-strait-april-may-2020.conf b/workflow/example/fram-strait-april-may-2020.conf new file mode 100644 index 00000000..f355783b --- /dev/null +++ b/workflow/example/fram-strait-april-may-2020.conf @@ -0,0 +1,9 @@ +START="2020-04-26" +END="2020-05-31" +SATELLITES="aqua", "terra" +LOCATION="fram-strait" +CRS="EPSG:3413" +BBOX="250000,-2100000,1100000,-400000" +SCALE=250 +CENTROID_LAT=76.9405 +CENTROID_LON=-16.631 \ No newline at end of file diff --git a/workflow/example/fram-strait-june.conf b/workflow/example/fram-strait-june.conf new file mode 100644 index 00000000..ca836fcb --- /dev/null +++ b/workflow/example/fram-strait-june.conf @@ -0,0 +1,9 @@ +START="2023-06-28" +END="2023-07-07" +SATELLITES="aqua", "terra" +LOCATION="fram-strait" +CRS="EPSG:3413" +BBOX="504250,-1050250,610250,-964500" +SCALE=250 +CENTROID_LAT=79.4015734600186 +CENTROID_LON=-16.05024562885536 \ No newline at end of file diff --git a/workflow/example/iftpipeline-test-case.conf b/workflow/example/iftpipeline-test-case.conf new file mode 100644 index 00000000..dd7bfc69 --- /dev/null +++ b/workflow/example/iftpipeline-test-case.conf @@ -0,0 +1,10 @@ +# The test case from the IFTPipeline.jl/test directory +START="2022-09-14" +END="2022-09-14" +SATELLITES="aqua", "terra" +LOCATION="ift-pipeline-test" +CRS="EPSG:3413" +BBOX="381500,-1002250,651000,-899250" +SCALE=250 +CENTROID_LAT=80.04 +CENTROID_LON=16.50 \ No newline at end of file diff --git a/workflow/example/ne-greenland.conf b/workflow/example/ne-greenland.conf new file mode 100644 index 00000000..38c1bdb8 --- /dev/null +++ b/workflow/example/ne-greenland.conf @@ -0,0 +1,9 @@ +START="2022-09-10" +END="2022-09-14" +SATELLITES="aqua", "terra" +LOCATION="ne-greenland" +CRS="EPSG:3413" +BBOX="381550,-1002475,651050,-899225" +SCALE=250 +CENTROID_LAT=80.04 +CENTROID_LON=16.50 \ No newline at end of file diff --git a/workflow/flow.cylc b/workflow/flow.cylc new file mode 100644 index 00000000..eede0f5e --- /dev/null +++ b/workflow/flow.cylc @@ -0,0 +1,280 @@ +#!jinja2 + +[scheduler] + allow implicit tasks = True + +[task parameters] + satellite = {{ SATELLITES | join(", ")}} + +[scheduling] + initial cycle point = {{ START }} + final cycle point = {{ END }} + runahead limit = P10 + [[graph]] + R1 = """ + INIT + init_soit => get_all_overpass_times + """ + P1D = """ + get_all_overpass_times[^] => get_single_overpass_time + INIT[^]:succeed-all => LOAD:succeed-all => PREPROCESS:succeed-all => exportH5 => done + done[-P1D] => done + + """ + R1/P0Y = done => tracking + + [[queues]] + [[[fetchdata]]] + limit = 4 + # Limit to 4 instances hitting the API at once. + # Values higher than this might lead to throttling. + members = LOAD_IMAGE + + [[[preprocess]]] + limit = 20 + # Limit to 2 instances only due to memory constraints. + # Can be removed on a system with more memory or a scheduler. + members = PREPROCESS + + +[runtime] + [[root]] + work sub-directory = . + execution time limit = PT5M + [[[directives]]] + --mem = 8G + + [[[environment]]] + # Update these variables with your run parameters + date=$(isodatetime "$CYLC_TASK_CYCLE_POINT" --print-format CCYY-MM-DD) + date_without_dashes=${date//-/} + all_overpass_time_file="overpass-times.csv" + + {% if IFT_INSTALL == "Local" %} + IFT="julia --project=/workspaces/ice-floe-tracker-workspace/ice-floe-tracker-pipeline/IFTPipeline.jl /workspaces/ice-floe-tracker-workspace/ice-floe-tracker-pipeline/IFTPipeline.jl/src/cli.jl" + {% elif IFT_INSTALL == "Docker" %} + IFT="docker run -v `pwd`:/app -w /app brownccv/icefloetracker-julia" + {% elif IFT_INSTALL == "Apptainer" %} + IFT="apptainer run docker://brownccv/icefloetracker-julia:pr-198" + {% elif IFT_INSTALL == "ApptainerLocal" %} + IFT="apptainer run /oscar/data/mmart119/jholla10/ice-floe-tracker-pipeline/runtime/image/iftp-pr-198.simg" + {% else %} + {{ raise('IFT_INSTALL not recognized.') }} + {% endif %} + + {% if FSDPROC_INSTALL == "Local" %} + FSDPROC="pipx run --spec /workspaces/ice-floe-tracker-workspace/ebseg fsdproc --debug" + {% elif FSDPROC_INSTALL == "Source" %} + FSDPROC="pipx run --spec git+https://github.com/WilhelmusLab/ebseg fsdproc --debug" + {% elif FSDPROC_INSTALL == "Docker" %} + {{ raise('FSDPROC_INSTALL `Docker` not implemented.') }} + {% elif FSDPROC_INSTALL == "Apptainer" %} + {{ raise('FSDPROC_INSTALL `Apptainer` not implemented.') }} + {% else %} + {{ raise('FSDPROC_INSTALL not recognized.') }} + {% endif %} + + {% if PASS_TIME_INSTALL == "Local" %} + {{ raise('PASS_TIME_INSTALL `Local` not implemented.') }} + {% elif PASS_TIME_INSTALL == "Source" %} + PASS_TIME = "pipx run --spec git+https://github.com/wilhelmuslab/ice-floe-tracker-pipeline@jghrefactor/C3-update-soit-to-use-ios8601-datetimes#egg=satellite-overpass-identification-tool&subdirectory=satellite-overpass-identification-tool soit" + {% elif PASS_TIME_INSTALL == "Docker" %} + {{ raise('PASS_TIME_INSTALL `Docker` not implemented.') }} + {% elif PASS_TIME_INSTALL == "Apptainer" %} + {{ raise('PASS_TIME_INSTALL `Apptainer` not implemented.') }} + {% else %} + {{ raise('PASS_TIME_INSTALL not recognized.') }} + {% endif %} + + {% if COLORIZE_INSTALL == "Local" %} + COLORIZE = "pipx run --spec /workspaces/ice-floe-tracker-workspace/ice-floe-tracker-pipeline/label-colorizer colorize" + {% elif COLORIZE_INSTALL == "Source" %} + COLORIZE = "pipx run --spec git+https://github.com/wilhelmuslab/ice-floe-tracker-pipeline@merge-attempt-1#egg=label-colorizer&subdirectory=label-colorizer colorize" + {% elif COLORIZE_INSTALL == "Docker" %} + {{ raise('COLORIZE_INSTALL `Docker` not implemented.') }} + {% elif COLORIZE_INSTALL == "Apptainer" %} + {{ raise('COLORIZE_INSTALL `Apptainer` not implemented.') }} + {% else %} + {{ raise('COLORIZE_INSTALL not recognized.') }} + {% endif %} + + JULIA_DEBUG="Main,IFTPipeline,IceFloeTracker" + + [[INIT]] + [[init_soit]] + inherit = INIT + script = """ + ${PASS_TIME} --help + """ + [[init_fsdproc]] + inherit = INIT + script = """ + ${FSDPROC} --help + """ + [[init_colorize]] + inherit = INIT + script = """ + ${COLORIZE} --help + """ + [[init_iftp]] + inherit = INIT + script = """ + ${IFT} --help + """ + execution time limit = PT25M + [[[directives]]] + --mem = 64G + + [[get_all_overpass_times]] + execution retry delays = PT15S, PT10M, PT1H, PT3H + script = """ + ${PASS_TIME} --csvoutpath ${all_overpass_time_file} \ + --startdate {{ START }} --enddate {{ END }} \ + --centroid-lat {{ CENTROID_LAT }} --centroid-lon {{ CENTROID_LON }} \ + --SPACEUSER {{ SPACEUSER }} --SPACEPSWD {{ SPACEPSWD }} + """ + + [[]] + pre-script = """ + mkdir -p ${img_prefix} + """ + [[[environment]]] + satellite=${CYLC_TASK_PARAM_satellite} + img_prefix = "${date_without_dashes}.{{ SCALE }}m.${satellite}/" + landmask_file = "${img_prefix}landmask.tiff" + landmask_binarized_file = "${img_prefix}landmask.binarized.tiff" + landmask_binarized_dilated_file = "${img_prefix}landmask.binarized.dilated.tiff" + truecolor_file = ${img_prefix}truecolor.tiff + falsecolor_file = ${img_prefix}falsecolor.tiff + cloud_file = ${img_prefix}cloud.tiff + labeled_file = ${img_prefix}labeled.tiff + colorized_labeled_file = ${img_prefix}labeled.colorized.tiff + labeled_props_file = ${img_prefix}labeled.props.csv + overpass_time_file = ${img_prefix}overpass.txt + hdf5_archive_file = ${img_prefix}results.h5 + + [[LOAD]] + + [[get_single_overpass_time]] + inherit = LOAD, + script = """ + # Get the date of the current task cycle point in YYYY-MM-DD format + date_=$(isodatetime "$CYLC_TASK_CYCLE_POINT" --print-format "CCYY-MM-DD") + + # -F,: Split the lines from the file on commas, + # -v search=...: set the value of the `search` variable to the date and satellite + # $0 ~ search: match the line in the file containing the search variable + # {print $3}: print the third value (from the comma-delimited split) + # > ${filename}: write the value out to file + awk -F, \ + -v search="${date_},${satellite}" \ + '$0 ~ search {print $3}' ${all_overpass_time_file} \ + > ${overpass_time_file} + """ + + [[LOAD_IMAGE]] + inherit=LOAD + execution retry delays = PT15S, PT10M, PT1H, PT3H + [[[environment]]] + fsdprocargs="--crs {{ CRS }} --bbox {{ BBOX }} --scale {{ SCALE }} --datetime ${date} --satellite ${satellite} " + + [[load_truecolor]] + inherit = LOAD_IMAGE, + script = """ + ${FSDPROC} load ${truecolor_file} --kind truecolor ${fsdprocargs} + """ + + [[load_falsecolor]] + inherit = LOAD_IMAGE, + script = """ + ${FSDPROC} load ${falsecolor_file} --kind bands721 ${fsdprocargs} + """ + + [[load_cloud]] + inherit = LOAD_IMAGE, + script = """ + ${FSDPROC} load ${cloud_file} --kind cloud ${fsdprocargs} + """ + + [[load_landmask]] + inherit = LOAD_IMAGE, + script=""" + ${FSDPROC} load ${landmask_file} --kind landmask ${fsdprocargs} + """ + + [[PREPROCESS]] + execution time limit = PT120M + execution retry delays = PT10M + + [[preprocess]] + inherit = PREPROCESS, + script = """ + {% if PREPROCESSING == "Lopez" %} + + ${IFT} landmask_single \ + -i ${landmask_file} \ + -o ${landmask_binarized_file} \ + -d ${landmask_binarized_dilated_file} + + ${IFT} preprocess_single \ + -t ${truecolor_file} \ + -r ${falsecolor_file} \ + -l ${landmask_binarized_file} \ + -d ${landmask_binarized_dilated_file} \ + -o ${labeled_file} + + ${IFT} extractfeatures_single \ + --input ${labeled_file} \ + --output ${labeled_props_file} \ + --minarea {{ MINFLOEAREA }} \ + --maxarea {{ MAXFLOEAREA }} + + {% elif PREPROCESSING == "Buckley" %} + + workdir="${labeled_file}.work" + mkdir -p "${workdir}" + ${FSDPROC} process ${truecolor_file} ${cloud_file} ${landmask_file} ${workdir} + cp ${workdir}/final.tif ${labeled_file} + cp ${workdir}/props.csv ${labeled_props_file} + + {% else %} + + {{ raise('PREPROCESSING type not recognized.') }} + + {% endif %} + + ${COLORIZE} ${labeled_file} ${colorized_labeled_file} + """ + + [[exportH5]] + inherit = None, + # Package intermediate and final outputs into HDF5 files + script = """ + ${IFT} makeh5files_single \ + --passtime `cat ${overpass_time_file}` \ + --truecolor ${truecolor_file} \ + --falsecolor ${falsecolor_file} \ + --labeled ${labeled_file} \ + --props ${labeled_props_file} \ + --output ${hdf5_archive_file} + """ + + [[tracking]] + execution retry delays = PT10M + execution time limit = PT120M + [[[directives]]] + --mem = 64G + # Pair and track identified floes across days + script = """ + images=(*/labeled.tiff) + props=${images[@]/.tiff/.props.csv} + landmasks=(*/landmask.tiff) + passtimes=${images[@]/labeled.tiff/overpass.txt} + + ${IFT} track_single \ + --imgs ${images[@]} \ + --props ${props[@]} \ + --latlon ${landmasks[0]} \ + --passtimes $(cat ${passtimes[@]} | tr '\n' ' ') \ + --output floes.tracked.csv + """ \ No newline at end of file diff --git a/workflow/meta/rose-meta.conf b/workflow/meta/rose-meta.conf new file mode 100644 index 00000000..a37f14d8 --- /dev/null +++ b/workflow/meta/rose-meta.conf @@ -0,0 +1,4 @@ +[template variables] +[template variables=PREPROCESSING] +values="Lopez", "Buckley" +type='character' \ No newline at end of file diff --git a/workflow/oscar.global.cylc b/workflow/oscar.global.cylc new file mode 100644 index 00000000..cdff1391 --- /dev/null +++ b/workflow/oscar.global.cylc @@ -0,0 +1,3 @@ +[platforms] + [[localhost]] + job runner = slurm \ No newline at end of file diff --git a/workflow/report/README.md b/workflow/report/README.md deleted file mode 100644 index 00620b5e..00000000 --- a/workflow/report/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Workflow Report Directory - -This directory contains any runtime reports generated by the workflow manager while running the pipeline. diff --git a/workflow/rose-suite.conf b/workflow/rose-suite.conf new file mode 100644 index 00000000..609c65a7 --- /dev/null +++ b/workflow/rose-suite.conf @@ -0,0 +1,29 @@ +[template variables] +START="2019-03-19" +END="2019-03-20" +SATELLITES="aqua", "terra" +LOCATION="beaufort_sea" +CRS="EPSG:3413" +BBOX="-2112500.0,-262500.0,-2012500.0,-162500.0" +SCALE=250 +CENTROID_LAT=71.02795656503653 +CENTROID_LON=-129.11755440990464 +MINFLOEAREA=350 +MAXFLOEAREA=90000 +PREPROCESSING="Lopez" # "Lopez" or "Buckley" + + +# Install locations: +# - "Apptainer" - use apptainer to load an image from the docker registry on the fly +# - "Docker" - use docker to load an image from the docker registry on the fly +# - "Source" - download and build the application from source on the fly, e.g. using pipx +# - "Local" - use a local installation on the current machine +# - "ApptainerLocal" – use a pre-selected apptainer image in ../runtime/images + +IFT_INSTALL="ApptainerLocal" # "Apptainer" or "Docker" or "Local" or "ApptainerLocal" +FSDPROC_INSTALL="Source" # "Apptainer" or "Docker" or "Source" or "Local" +PASS_TIME_INSTALL="Source" # "Apptainer" or "Docker" or "Source" or "Local" +COLORIZE_INSTALL="Source" # "Apptainer" or "Docker" or "Source" or "Local" + +SPACEUSER="john_holland1@brown.edu" +SPACEPSWD="iAwyPF8f5XkBWXe" diff --git a/workflow/scripts/Dockerfile b/workflow/scripts/Dockerfile deleted file mode 100644 index b2c83cc5..00000000 --- a/workflow/scripts/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM python:3.9-bullseye - -ENV TERM=xterm - -# DEPENDENCIES -#=========================================== -RUN apt-get clean && apt-get update && \ - apt-get install -y wget python3-pip && \ - rm -rf /var/lib/apt/list/* -RUN apt-get install gdal-bin -y -RUN pip3 install skyfield \ - requests - -# Build software -#=========================================== -COPY fetchdata.sh /usr/local/bin -COPY pass_time_cylc.py /usr/local/bin -RUN chmod a+x /usr/local/bin/pass_time_cylc.py -RUN chmod a+x /usr/local/bin/fetchdata.sh -CMD [ "/bin/bash", "-c" ] diff --git a/workflow/scripts/fetchdata.sh b/workflow/scripts/fetchdata.sh deleted file mode 100755 index cf93b39b..00000000 --- a/workflow/scripts/fetchdata.sh +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -PROGRAM_NAME="$(basename "${0}")" - -BOLD="$(tput bold)" -NORMAL="$(tput sgr0)" -TAB="$(printf "\t")" - -PROJ_WGS84='+proj=longlat +datum=WGS84 +no_defs' -PROJ_EPSG3413='+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs' - -GDAL_SRS='+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=m +no_defs' -GDAL_WMS_TEMPLATE=' - - https://gibs.earthdata.nasa.gov/wmts/epsg3413/best/%%LAYER%%/default/%%DATE%%/250m/${z}/${y}/${x}.%%EXT%% - - - -4194304 - 4194304 - 4194304 - -4194304 - 5 - 2 - 2 - top - - EPSG:3413 - 512 - 512 - 3 -' - -warn() { - >&2 echo "$@" -} - -die() { - local message="$1" - local status="${2:-1}" - - warn "${message}" - exit "${status}" -} - -usage() { - cat < /dev/null -} - -download_truecolor() { - local bounding_box="${1}" - local date="${2}" - local enddate="${3}" - local output="${4}" - local layer filename xml - local ext="jpeg" - - - echo "downloading true color images" - while [ "${date}" != "${enddate}" ]; do - for sat in Aqua Terra; do - layer="MODIS_${sat}_CorrectedReflectance_TrueColor" - filename="$(echo "${date}" | sed -e 's/-//g').$(echo "${sat}" | tr '[:upper:]' '[:lower:]').truecolor.250m.tiff" - xml="$(echo "${GDAL_WMS_TEMPLATE}" | sed -e "s/%%LAYER%%/${layer}/" -e "s/%%EXT%%/${ext}/" -e "s/%%DATE%%/${date}/" )" - - gdalwarp -overwrite -t_srs "${GDAL_SRS}" -te ${bounding_box} "${xml}" "${output}/${filename}" &> /dev/null - done - - date="$(add_day "${date}")" - printf '.' - done - - printf '\n' -} - -download_falsecolor() { - local bounding_box="${1}" - local date="${2}" - local enddate="${3}" - local output="${4}" - - local layer filename xml - local ext="jpeg" - - - echo "downloading falsecolor color images" - while [ "${date}" != "${enddate}" ]; do - - for sat in Aqua Terra; do - layer="MODIS_${sat}_CorrectedReflectance_Bands721" - filename="$(echo "${date}" | sed -e 's/-//g').$(echo "${sat}" | tr '[:upper:]' '[:lower:]').falsecolor.250m.tiff" - - xml="$(echo "${GDAL_WMS_TEMPLATE}" | sed -e "s/%%LAYER%%/${layer}/" -e "s/%%EXT%%/${ext}/" -e "s/%%DATE%%/${date}/" )" - gdalwarp -overwrite -t_srs "${GDAL_SRS}" -te ${bounding_box} "${xml}" "${output}/${filename}" &> /dev/null - done - - date="$(add_day "${date}")" - printf '.' - done - - printf '\n' -} - -main() { - local crs='wgs84' - local startdate="$(date "+%Y-%m-%d")" - local enddate="$(date "+%Y-%m-%d")" - local output='.' - local raw_bounding_box='' - - local opt - while getopts ":b:c:e:ho:s:" opt; do - case $opt in - b) - raw_bounding_box="${OPTARG}" - ;; - c) - crs="${OPTARG}" - ;; - e) - enddate="${OPTARG}" - ;; - h) - usage; exit - ;; - o) - output="${OPTARG}" - ;; - s) - startdate="${OPTARG}" - ;; - :) - die "option expected for \"-${OPTARG}\"" - ;; - ?) - die "invalid option: \"${OPTARG}\"" - ;; - esac - done - - if [ "${crs}" != "wgs84" ] && [ "${crs}" != "epsg3413" ]; then - die "unknown coordinate reference system: \"${crs}\"" - fi - - if [ "x${raw_bounding_box}" = 'x' ]; then - die "undefined bounding box" - fi - - # split raw bounding box into top left and bottom right sets - local topleft="$(get_topleft "${raw_bounding_box}")" - local bottomright="$(get_bottomright "${raw_bounding_box}")" - local x1 y1 x2 y2 - - # convert lat/lon to polar stereographic - if [ "${crs}" == "wgs84" ]; then - topleft="$(convert_to_epsg3413 "${topleft}")" - bottomright="$(convert_to_epsg3413 "${bottomright}")" - fi - - # assign x and y from top left and bottom right coordinate sets - x1="$(echo "${topleft}" | awk '{ print $1 }')" - y1="$(echo "${topleft}" | awk '{ print $2 }')" - x2="$(echo "${bottomright}" | awk '{ print $1 }')" - y2="$(echo "${bottomright}" | awk '{ print $2 }')" - - local bounding_box - - bounding_box="$(sort_xy $x1 $y1 $x2 $y2)" - - # make standard output dirs - mkdir -p "${output}" - mkdir -p "${output}/falsecolor" - mkdir -p "${output}/truecolor" - - # download the images - download_landmask "${bounding_box}" "${startdate}" "${output}" - download_truecolor "${bounding_box}" "${startdate}" "${enddate}" "${output}/truecolor" - download_falsecolor "${bounding_box}" "${startdate}" "${enddate}" "${output}/falsecolor" -} - -main "$@" diff --git a/workflow/scripts/flow_generator.py b/workflow/scripts/flow_generator.py deleted file mode 100644 index 71a1cd70..00000000 --- a/workflow/scripts/flow_generator.py +++ /dev/null @@ -1,104 +0,0 @@ - -from jinja2 import Environment, FileSystemLoader # templating engine -import pandas as pd, os, argparse - -def get_parameters(data, name, crs=None): - if name != "bounding_box": - return ",".join(list(data[name].values.astype(str))) - elif name == "bounding_box" and crs is not None: - return get_bounding_box_list(data, crs) - -def generate_bounding_box(row, crs): - if crs == "epsg3413": - columns = ["left_x", "top_y", "right_x", "lower_y"] - elif crs == "wgs84": - columns = ["top_left_lat", "top_left_lon", "lower_right_lat", "lower_right_lon"] - return "@".join(list(row[columns].values.astype(str))) - -def get_bounding_box_list(data, crs="wgs84"): - f = lambda x: generate_bounding_box(x, crs) - return ",".join(data.apply(f, axis=1).values) - -def generate_cylc_file(csvfile="site_locations.csv", template="flow_template.j2", - template_dir="./config", crs="wgs84", minfloearea=350, maxfloearea=90_000): - - df = pd.read_csv(csvfile) - date_columns = ['startdate', 'enddate'] - df[date_columns] = df[date_columns].apply(lambda x: pd.to_datetime(x).dt.strftime('%Y-%m-%d')) - df['bounding_box'] = None - data = {c: get_parameters(df, c, crs) for c in df.columns} - data['rows'] = len(df.index) - 1 - data['crs'] = crs - data['minfloearea'] = minfloearea - data['maxfloearea'] = maxfloearea - data['centroid_x'] = data['center_lat'] - data['centroid_y'] = data['center_lon'] - env = Environment(loader=FileSystemLoader(template_dir)) - template = env.get_template(template) - fname = os.path.join(template_dir, "flow.cylc") - content = template.render(data) - with open(fname, "w") as fh: - fh.write(content) - - -def main(): - example = """Example: - - The following would create a flow file for however many rows of data the user input into the - site_location.csv file for use on a high-performance cluster (like Brown's Oscar) with polar - stereographic coordinates, and intending to process floes between 350 and 75000 pixels in area. - - python ./workflow/scripts/flow_generator.py --csvfile "./config/site_locations.csv" --template "flow_template_hpc.j2" --template_dir "./config/cylc_hpc" --crs "epsg3413" --minfloearea 350 --maxfloearea 75000 - - In this second example, a flow file will be generated for use on a local OS using lat/lon inputs - and considering floes from 500 - 90000 pixels in area. - - python workflow/scripts/flow_generator.py --csvfile "./config/site_locations.csv" --template "flow_template_local.j2" --template_dir "./config/cylc_local" --crs "wgs84" --minfloearea 500 """ - - parser = argparse.ArgumentParser( - description="Generate a Cylc flow file from a CSV matrix of unique location-parameter sets", - epilog=example, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument( - "--csvfile", - type=str, - help="path to `site_locations.csv` file with each param set as unique row", - ) - parser.add_argument( - "--template", - type=str, - choices=['flow_template_hpc.j2','flow_template_local.j2'], - help="`flow_template.j2` file that needs to be populated", - ) - parser.add_argument( - "--template_dir", - type=str, - choices=['./config/cylc_hpc','./config/cylc_local'], - help="Location path to the directory containing j2 template file", - ) - parser.add_argument( - "--crs", - type=str, - choices=['wgs84','epsg3413'], - help="Either 'wgs84' or 'epsg3413'", - ) - parser.add_argument( - "--minfloearea", - type=int, - default=350, - help="minimum pixel size of ice floes to process", - ) - parser.add_argument( - "--maxfloearea", - type=int, - default=90000, - help="maximum pixel size of ice floes to process" - ) - - args = parser.parse_args() - - generate_cylc_file(**vars(args)) - -if __name__ == "__main__": - - main() diff --git a/workflow/scripts/ice-floe-tracker.jl b/workflow/scripts/ice-floe-tracker.jl deleted file mode 100755 index 9877d2e3..00000000 --- a/workflow/scripts/ice-floe-tracker.jl +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env julia - -using ArgParse -using LoggingExtras -using IceFloeTracker -using IFTPipeline -using IFTPipeline: mkclipreprocess!, mkcliextract!, mkclitrack!, mkclilandmask!, mkcli! -using Serialization - - -""" - setuplogger(option::Int64, path::String) - -Setup logger for the ice floe tracker. If `option` is 0, log to file only. If `option` is 1, log to file and terminal. -""" -function setuplogger(option::Int64, command::Symbol) - output = joinpath(@__DIR__, "..", "report") - cmd = string(command) - - filelogger = FileLogger(joinpath(output, "$cmd-logfile.log")) # add command prefix to logfile name - - # filter out debug messages - filtlogger = EarlyFilteredLogger(filelogger) do args - r = Logging.Info <= args.level <= Logging.Warn && args._module === IFTPipeline - return r - end - - if option == 0 - return TeeLogger(filtlogger, - ) - elseif option == 1 - return TeeLogger( - global_logger(), - filtlogger - ) - end -end - -function main(args) - settings = ArgParseSettings(; autofix_names=true) - - @add_arg_table! settings begin - "landmask" - help = "Generate land mask images" - action = :command - - "preprocess" - help = "Preprocess truecolor/falsecolor images" - action = :command - - "extractfeatures" - help = "Extract ice floe features from segmented floe image" - action = :command - - "makeh5files" - help = "Make HDF5 files from extracted floe features" - action = :command - - "track" - help = "Pair ice floes in day k with ice floes in day k+1" - action = :command - end - - command_common_args = [ - "--log", - Dict(:help => "Show log on terminal; either 1 or 0", :required => false, :arg_type => Int, - :default => 0, :range_tester => (x -> x == 0 || x == 1) - )] - - mkcli!(settings, command_common_args) - - parsed_args = parse_args(args, settings; as_symbols=true) - - command = parsed_args[:_COMMAND_] - command_args = parsed_args[command] - command_func = getfield(IFTPipeline, Symbol(command)) - logoption = command_args[:log] - - # delete log option from command_args so it doesn't get passed to command_func - delete!(command_args, :log) - - logger = setuplogger(logoption, command) - - with_logger(logger) do - @time command_func(; command_args...) - end - return nothing -end - -main(ARGS)