Merge pull request #6 from emit-sds/imgspec

ImgSPEC Merge

.imgspec/algorithm_config.yaml

@@ -0,0 +1,41 @@
+# DO NOT DELETE
+# THIS CONFIG IS AUTO-GENERATED BY ADE UI
+algo_name: SpectralUnmixing
+version: 0.1.0
+environment: ubuntu
+repository_url: https://github.com/emit-sds/SpectralUnmixing.git
+docker_url: registry.imgspec.org/root/ade_base_images/vanilla:1.0
+
+# fill out these fields
+# explain what this algorithm does
+description: A general, fast, and flexible package for spectral unmixing.
+# path to the build script for installing and building the algorithm in the docker image
+build_command: SpectralUnmixing/.imgspec/install.sh
+# path to the wrapper script for running the algorithm
+run_command: SpectralUnmixing/.imgspec/imgspec_run.sh
+# set a storage value in GB or MB or KB, e.g. "100GB", "20MB", "10KB"
+disk_space: 80GB
+inputs:
+# remove below this line if no inputs
+# rename and set algorithm input names accordingly
+  - name: reflectance_granule
+    type: file
+  - name: endmember_library
+    type: positional
+  - name: endmember_column
+    type: positional
+    default: "class"
+  - name: n_cores
+    type: positional
+    default: "1"
+  - name: refl_nodata
+    type: positional
+    default: "None"
+  - name: refl_scale
+    type: positional
+    default: "None"
+  - name: normalization
+    type: positional
+    default: "None"
+
+

.imgspec/get_paths_from_granules.py

@@ -0,0 +1,56 @@
+#! /usr/bin/python
+
+import argparse
+import glob
+import os
+import sys
+import tarfile
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    products = ["rfl"]
+    formats = ["envi"]
+    parser.add_argument("-p", "--product",
+                        help=("Choose one of the following product types: " + ", ".join(products)))
+    parser.add_argument("-f", "--format",
+                        help=("Choose one of the following formats: " + ", ".join(formats)))
+    args = parser.parse_args()
+
+    if args.product:
+        if args.product not in products:
+            print("ERROR: Product \"%s\" is not a valid product choice." % args.product)
+            sys.exit(1)
+    if args.format:
+        if args.format not in formats:
+            print("ERROR: Format \"%s\" is not a valid format choice." % args.format)
+            sys.exit(1)
+    return args
+
+
+def main():
+    args = parse_args()
+
+    # Unzip and untar granules
+    input_dir = "input"
+    granule_paths = glob.glob(os.path.join(input_dir, "*.tar.gz"))
+    for g in granule_paths:
+        tar_file = tarfile.open(g)
+        tar_file.extractall(input_dir)
+        tar_file.close()
+        os.remove(g)
+
+    # Get paths based on product type file matching
+    paths = []
+    if args.product == "rfl":
+        # AVIRIS SDS uses *rfl*img and *corr*img for distributed files
+        paths = glob.glob(os.path.join(input_dir, "*", "*rfl*img"))
+        paths += glob.glob(os.path.join(input_dir, "*", "*corr*img"))
+        # ISOFIT uses *rfl for processed reflectance files
+        paths += glob.glob(os.path.join(input_dir, "*", "*rfl"))
+
+    print(",".join(paths))
+
+
+if __name__ == "__main__":
+    main()

.imgspec/imgspec_run.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# Description:
+#
+# The top-level run script to execute unmix.jl on ImgSPEC. This script accepts the inputs described in the
+# algorithm_config.yaml file and pre-processes them as needed to pass into SpectralUnmixing/unmix.jl.  This script
+# is currently compatible with AVIRIS Classic, AVIRIS-NG, and PRISMA data.
+#
+# Inputs:
+#
+# $1: URL of endmember library
+# $2: Column name from endmember library that describes the library classes (default is "class")
+# $3: Number of cores to use (default is "1")
+# $4: Nodata value expected in input reflectance data (default is "None" - will use the repo's default)
+# $5: Scale image data (divide it by) this amount (default is "None" - will use the repo's default)
+# $6: Flag to indicate the scaling type. Options = [none, brightness, specific wavelength] (default is "None" - will
+#     use the repo's default)
+#
+# In addition to the positional arguments, this script expects a downloaded reflectance granule to be present in a
+# folder called "input".
+
+# Define some useful directories
+imgspec_dir=$( cd "$(dirname "$0")" ; pwd -P )
+specun_dir=$(dirname ${imgspec_dir})
+input="input"
+mkdir -p output
+
+# Activate conda environment
+source activate spectral-unmixing
+
+# Export JULIA_PROJECT again in case it doesn't carry over from install.sh
+export JULIA_PROJECT=$specun_dir
+
+# Spectral Unmixing paths
+unmix_exe="$specun_dir/unmix.jl"
+endmember_library_path="$input/endmember_library.csv"
+
+# Get reflectance path from input granule
+echo "Looking for input granule gzip and extracting to get reflectance path..."
+rfl_path=$(python ${imgspec_dir}/get_paths_from_granules.py -p rfl)
+echo "Found input reflectance file: $rfl_path"
+
+# Process positional args to get EcoSIS CSV files
+echo "Getting endmember library..."
+curl --retry 10 --output $endmember_library_path $1
+
+# Get output base from reflectance path
+rfl_name=$(basename $rfl_path)
+output_base="output"
+if [[ $rfl_name == f* ]]; then
+    output_base=$(echo $rfl_name | cut -c1-16)
+elif [[ $rfl_name == ang* ]]; then
+    output_base=$(echo $rfl_name | cut -c1-18)
+elif [[ $rfl_name == PRS* ]]; then
+    output_base=$(echo $rfl_name | cut -c1-38)
+fi
+output_base_path="output/$output_base"
+echo "Output base path: $output_base_path"
+
+# Build command and execute
+cmd="julia"
+
+# Add number of cores
+if [[ $3 != "1" ]]; then
+    cmd="$cmd -p $3"
+fi
+
+# Add the required args (and assume mode is sma for now)
+cmd="$cmd $unmix_exe $rfl_path $endmember_library_path $2 $output_base_path --mode=sma"
+
+# Add the optional args
+if [[ $4 != "None" ]]; then
+    cmd="$cmd --refl_nodata=$4"
+fi
+if [[ $5 != "None" ]]; then
+    cmd="$cmd --refl_scale=$5"
+fi
+if [[ $6 != "None" ]]; then
+    cmd="$cmd --normalization=$6"
+fi
+
+echo "Executing command: $cmd"
+$cmd

.imgspec/install.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Install script to 1) install Julia via conda, and 2) install the Julia dependencies for this project
+
+set -x
+
+# Define some useful directories
+imgspec_dir=$( cd "$(dirname "$0")" ; pwd -P )
+specun_dir=$(dirname ${imgspec_dir})
+
+# Install Julia and then install Julia dependencies
+conda create -n spectral-unmixing -y -c conda-forge julia=1.7 python=3.9
+source activate spectral-unmixing
+pushd $specun_dir
+julia -e 'using Pkg; Pkg.activate("."); Pkg.add(path="https://github.com/kmsquire/ArgParse2.jl"); Pkg.instantiate()'
+export JULIA_PROJECT=$specun_dir
+popd