diff --git a/.imgspec/algorithm_config.yaml b/.imgspec/algorithm_config.yaml new file mode 100644 index 0000000..93c6c5a --- /dev/null +++ b/.imgspec/algorithm_config.yaml @@ -0,0 +1,41 @@ +# DO NOT DELETE +# THIS CONFIG IS AUTO-GENERATED BY ADE UI +algo_name: SpectralUnmixing +version: 0.1.0 +environment: ubuntu +repository_url: https://github.com/emit-sds/SpectralUnmixing.git +docker_url: registry.imgspec.org/root/ade_base_images/vanilla:1.0 + +# fill out these fields +# explain what this algorithm does +description: A general, fast, and flexible package for spectral unmixing. +# path to the build script for installing and building the algorithm in the docker image +build_command: SpectralUnmixing/.imgspec/install.sh +# path to the wrapper script for running the algorithm +run_command: SpectralUnmixing/.imgspec/imgspec_run.sh +# set a storage value in GB or MB or KB, e.g. "100GB", "20MB", "10KB" +disk_space: 80GB +inputs: +# remove below this line if no inputs +# rename and set algorithm input names accordingly + - name: reflectance_granule + type: file + - name: endmember_library + type: positional + - name: endmember_column + type: positional + default: "class" + - name: n_cores + type: positional + default: "1" + - name: refl_nodata + type: positional + default: "None" + - name: refl_scale + type: positional + default: "None" + - name: normalization + type: positional + default: "None" + + diff --git a/.imgspec/get_paths_from_granules.py b/.imgspec/get_paths_from_granules.py new file mode 100644 index 0000000..b712731 --- /dev/null +++ b/.imgspec/get_paths_from_granules.py @@ -0,0 +1,56 @@ +#! /usr/bin/python + +import argparse +import glob +import os +import sys +import tarfile + + +def parse_args(): + parser = argparse.ArgumentParser() + products = ["rfl"] + formats = ["envi"] + parser.add_argument("-p", "--product", + help=("Choose one of the following product types: " + ", ".join(products))) + parser.add_argument("-f", "--format", + help=("Choose one of the following formats: " + ", ".join(formats))) + args = parser.parse_args() + + if args.product: + if args.product not in products: + print("ERROR: Product \"%s\" is not a valid product choice." % args.product) + sys.exit(1) + if args.format: + if args.format not in formats: + print("ERROR: Format \"%s\" is not a valid format choice." % args.format) + sys.exit(1) + return args + + +def main(): + args = parse_args() + + # Unzip and untar granules + input_dir = "input" + granule_paths = glob.glob(os.path.join(input_dir, "*.tar.gz")) + for g in granule_paths: + tar_file = tarfile.open(g) + tar_file.extractall(input_dir) + tar_file.close() + os.remove(g) + + # Get paths based on product type file matching + paths = [] + if args.product == "rfl": + # AVIRIS SDS uses *rfl*img and *corr*img for distributed files + paths = glob.glob(os.path.join(input_dir, "*", "*rfl*img")) + paths += glob.glob(os.path.join(input_dir, "*", "*corr*img")) + # ISOFIT uses *rfl for processed reflectance files + paths += glob.glob(os.path.join(input_dir, "*", "*rfl")) + + print(",".join(paths)) + + +if __name__ == "__main__": + main() diff --git a/.imgspec/imgspec_run.sh b/.imgspec/imgspec_run.sh new file mode 100755 index 0000000..61a9193 --- /dev/null +++ b/.imgspec/imgspec_run.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Description: +# +# The top-level run script to execute unmix.jl on ImgSPEC. This script accepts the inputs described in the +# algorithm_config.yaml file and pre-processes them as needed to pass into SpectralUnmixing/unmix.jl. This script +# is currently compatible with AVIRIS Classic, AVIRIS-NG, and PRISMA data. +# +# Inputs: +# +# $1: URL of endmember library +# $2: Column name from endmember library that describes the library classes (default is "class") +# $3: Number of cores to use (default is "1") +# $4: Nodata value expected in input reflectance data (default is "None" - will use the repo's default) +# $5: Scale image data (divide it by) this amount (default is "None" - will use the repo's default) +# $6: Flag to indicate the scaling type. Options = [none, brightness, specific wavelength] (default is "None" - will +# use the repo's default) +# +# In addition to the positional arguments, this script expects a downloaded reflectance granule to be present in a +# folder called "input". + +# Define some useful directories +imgspec_dir=$( cd "$(dirname "$0")" ; pwd -P ) +specun_dir=$(dirname ${imgspec_dir}) +input="input" +mkdir -p output + +# Activate conda environment +source activate spectral-unmixing + +# Export JULIA_PROJECT again in case it doesn't carry over from install.sh +export JULIA_PROJECT=$specun_dir + +# Spectral Unmixing paths +unmix_exe="$specun_dir/unmix.jl" +endmember_library_path="$input/endmember_library.csv" + +# Get reflectance path from input granule +echo "Looking for input granule gzip and extracting to get reflectance path..." +rfl_path=$(python ${imgspec_dir}/get_paths_from_granules.py -p rfl) +echo "Found input reflectance file: $rfl_path" + +# Process positional args to get EcoSIS CSV files +echo "Getting endmember library..." +curl --retry 10 --output $endmember_library_path $1 + +# Get output base from reflectance path +rfl_name=$(basename $rfl_path) +output_base="output" +if [[ $rfl_name == f* ]]; then + output_base=$(echo $rfl_name | cut -c1-16) +elif [[ $rfl_name == ang* ]]; then + output_base=$(echo $rfl_name | cut -c1-18) +elif [[ $rfl_name == PRS* ]]; then + output_base=$(echo $rfl_name | cut -c1-38) +fi +output_base_path="output/$output_base" +echo "Output base path: $output_base_path" + +# Build command and execute +cmd="julia" + +# Add number of cores +if [[ $3 != "1" ]]; then + cmd="$cmd -p $3" +fi + +# Add the required args (and assume mode is sma for now) +cmd="$cmd $unmix_exe $rfl_path $endmember_library_path $2 $output_base_path --mode=sma" + +# Add the optional args +if [[ $4 != "None" ]]; then + cmd="$cmd --refl_nodata=$4" +fi +if [[ $5 != "None" ]]; then + cmd="$cmd --refl_scale=$5" +fi +if [[ $6 != "None" ]]; then + cmd="$cmd --normalization=$6" +fi + +echo "Executing command: $cmd" +$cmd diff --git a/.imgspec/install.sh b/.imgspec/install.sh new file mode 100755 index 0000000..9da1a1c --- /dev/null +++ b/.imgspec/install.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Install script to 1) install Julia via conda, and 2) install the Julia dependencies for this project + +set -x + +# Define some useful directories +imgspec_dir=$( cd "$(dirname "$0")" ; pwd -P ) +specun_dir=$(dirname ${imgspec_dir}) + +# Install Julia and then install Julia dependencies +conda create -n spectral-unmixing -y -c conda-forge julia=1.7 python=3.9 +source activate spectral-unmixing +pushd $specun_dir +julia -e 'using Pkg; Pkg.activate("."); Pkg.add(path="https://github.com/kmsquire/ArgParse2.jl"); Pkg.instantiate()' +export JULIA_PROJECT=$specun_dir +popd