go: impl nldi and xy flags; remove thirdparty dep; r: update subset p…

…roto

.github/workflows/Generate-subset-app.yaml

@@ -25,21 +25,21 @@ jobs:
       - name: Package MacOS executable
         working-directory: ./cli
         run: |
-          GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build
+          GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -o hfsubset
           tar -czvf hfsubset-macos_amd64.tar.gz hfsubset
           rm hfsubset
 
       - name: Package Linux executable
         working-directory: ./cli
         run: |
-          GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build
+          GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o hfsubset
           tar -czvf hfsubset-linux_amd64.tar.gz hfsubset
           rm hfsubset
   
       - name: Package Windows executable
         working-directory: ./cli
         run: |
-          GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build
+          GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o hfsubset
           zip hfsubset-windows_amd64.zip hfsubset.exe
           rm hfsubset.exe
 

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "go.toolsEnvVars": {
+        "GO111MODULE": "on"
+    }
+}

cli/go.mod

@@ -5,9 +5,9 @@ go 1.20
 require github.com/schollz/progressbar/v3 v3.13.1
 
 require (
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/term v0.8.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
 )

cli/go.sum

@@ -5,6 +5,8 @@ github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -21,6 +23,10 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=

cli/main.go

@@ -26,11 +26,11 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
-
-	"github.com/schollz/progressbar/v3"
 )
 
 const usage string = `hfsubset - Hydrofabric Subsetter
@@ -40,14 +40,18 @@ Usage:
   hfsubset (-h | --help)
 
 Examples:
-  hfsubset -l divides,nexus        \
-           -o ./divides_nexus.gpkg \
-           -r "pre-release"        \
-           -t hl_uri                   \
-           "Gages-06752260"
-
+  # Using a hydrologic location identifier
   hfsubset -o ./poudre.gpkg -t hl_uri "Gages-06752260"
 
+  # Using network-linked data index identifiers
+  hfsubset -o ./poudre.gpkg -t nldi_feature "nwis:USGS-08279500"
+  
+  # Specifying layers and hydrofabric version
+  hfsubset -l divides,nexus -o ./divides_nexus.gpkg -r "pre-release" -t hl_uri "Gages-06752260"
+  
+  # Finding data around a POI
+  hfsubset -l flowpaths,reference_flowpaths -o ./sacramento_flowpaths.gpkg -t xy -121.494400,38.581573
+
 Options:
 `
 
@@ -63,6 +67,7 @@ type SubsetResponse struct {
 	data []byte
 }
 
+// Parse comma-delimited layers string
 func (opts *SubsetRequest) Layers() []string {
 	split := strings.Split(*opts.layers, ",")
 	for i, v := range split {
@@ -71,10 +76,53 @@ func (opts *SubsetRequest) Layers() []string {
 	return split
 }
 
-func (opts *SubsetRequest) IDs() []string {
+// Parse IDs format, i.e. trim spaces
+func (opts *SubsetRequest) IDs(key string) []string {
 	for i, v := range opts.id {
 		opts.id[i] = strings.TrimSpace(v)
 	}
+
+	if key == "nldi_feature" {
+		var feat struct {
+			FeatureSource string `json:"featureSource"`
+			FeatureId     string `json:"featureId"`
+		}
+
+		feat.FeatureSource = ""
+		feat.FeatureId = ""
+		for i, v := range opts.id {
+			f := strings.Split(v, ":")
+
+			feat.FeatureSource = f[0]
+			feat.FeatureId = f[1]
+			fstr, _ := json.Marshal(feat)
+			opts.id[i] = string(fstr)
+			feat.FeatureSource = ""
+			feat.FeatureId = ""
+		}
+	}
+
+	if key == "xy" {
+		var xy struct {
+			X float64
+			Y float64
+		}
+
+		xy.X = -1
+		xy.Y = -1
+		for i, v := range opts.id {
+			f := strings.Split(v, ",")
+
+			xy.X, _ = strconv.ParseFloat(f[0], 64)
+			xy.Y, _ = strconv.ParseFloat(f[1], 64)
+
+			fstr, _ := json.Marshal(xy)
+			opts.id[i] = string(fstr)
+			xy.X = -1
+			xy.Y = -1
+		}
+	}
+
 	return opts.id
 }
 
@@ -85,32 +133,25 @@ func (opts *SubsetRequest) MarshalJSON() ([]byte, error) {
 	switch *opts.id_type {
 	case "id":
 		key = "id"
-		break
 	case "hl_uri":
 		key = "hl_uri"
-		break
 	case "comid":
 		key = "comid"
-		break
 	case "nldi_feature":
-		// key = "nldi"
-		// break
-		fallthrough
+		key = "nldi_feature"
 	case "xy":
-		// key = "loc"
-		// break
-		panic("-nldi_feature and -xy support are not implemented currently")
+		key = "xy"
 	default:
 		panic("type " + *opts.id_type + " not supported; only one of: id, hl_uri, comid, nldi_feature, xy")
 	}
 
 	jsonmap["layers"] = opts.Layers()
-	jsonmap[key] = opts.IDs()
-	jsonmap["version"] = "pre-release"
+	jsonmap[key] = opts.IDs(key)
+	jsonmap["version"] = *opts.version
 	return json.Marshal(jsonmap)
 }
 
-func makeRequest(lambda_endpoint string, opts *SubsetRequest, bar *progressbar.ProgressBar) *SubsetResponse {
+func makeRequest(lambda_endpoint string, opts *SubsetRequest, logger *log.Logger) *SubsetResponse {
 	var uri string = lambda_endpoint + "/2015-03-31/functions/function/invocations"
 	payload, err := opts.MarshalJSON()
 	if err != nil {
@@ -119,14 +160,14 @@ func makeRequest(lambda_endpoint string, opts *SubsetRequest, bar *progressbar.P
 
 	reader := bytes.NewReader(payload)
 
-	bar.Describe("[1/4] waiting for response")
+	logger.Println("[1/4] waiting for response")
 	req, err := http.Post(uri, "application/json", reader)
 	if err != nil {
 		panic(err)
 	}
 	defer req.Body.Close()
 
-	bar.Describe("[2/4] reading hydrofabric subset")
+	logger.Println("[2/4] reading hydrofabric subset")
 	resp := new(SubsetResponse)
 	b := new(bytes.Buffer)
 	buffer := bufio.NewWriter(b)
@@ -136,15 +177,15 @@ func makeRequest(lambda_endpoint string, opts *SubsetRequest, bar *progressbar.P
 	}
 
 	r := b.Bytes()
+
+	// Trim quotes if returned
 	if r[0] == '"' && r[len(r)-1] == '"' {
 		r = r[1 : len(r)-1]
 	}
 
-	bar.Describe("[3/4] decoding gzip")
+	logger.Println("[3/4] parsing base64 response")
 	rr := bytes.NewReader(r)
 	gpkg := base64.NewDecoder(base64.StdEncoding, rr)
-	// gpkg, _ := gzip.NewReader(rr)
-	// defer gpkg.Close()
 	resp.data, err = io.ReadAll(gpkg)
 	if err != nil {
 		panic(err)
@@ -153,16 +194,15 @@ func makeRequest(lambda_endpoint string, opts *SubsetRequest, bar *progressbar.P
 	return resp
 }
 
-func writeToFile(request *SubsetRequest, response *SubsetResponse, bar *progressbar.ProgressBar) int {
+func writeToFile(request *SubsetRequest, response *SubsetResponse, logger *log.Logger) int {
 	f, err := os.Create(*request.output)
 	if err != nil {
 		panic(err)
 	}
 
-	bar.Describe(fmt.Sprintf("[4/4] writing to %s", *request.output))
+	logger.Printf("[4/4] writing to %s", *request.output)
 	w := bufio.NewWriter(f)
-	mw := io.MultiWriter(w, bar)
-	n, err := mw.Write(response.data)
+	n, err := w.Write(response.data)
 	if err != nil {
 		panic(err)
 	}
@@ -204,12 +244,11 @@ Either "all" or "core", or one or more of:
 		*opts.layers = "divides,nexus,flowpaths,network,hydrolocations"
 	}
 
-	bar := progressbar.NewOptions(3,
-		progressbar.OptionSetWidth(15),
-		progressbar.OptionSetDescription("[0/4] sending http request"),
-		progressbar.OptionShowBytes(false),
-		progressbar.OptionSetVisibility(!*quiet),
-	)
+	logger := log.New(os.Stdout, "hfsubset ==> ", log.Ltime)
+	if *quiet {
+		logger.SetOutput(io.Discard)
+	}
+	logger.Println("[0/4] sending http request")
 
 	var endpoint string
 	if v, ok := os.LookupEnv("HFSUBSET_ENDPOINT"); ok {
@@ -219,11 +258,9 @@ Either "all" or "core", or one or more of:
 		endpoint = "https://hfsubset-e9kvx.ondigitalocean.app"
 	}
 
-	resp := makeRequest(endpoint, opts, bar)
+	resp := makeRequest(endpoint, opts, logger)
 	response_size := len(resp.data)
-	bytes_written := writeToFile(opts, resp, bar)
-	bar.Finish()
-	println() // so progress bar doesn't show up
+	bytes_written := writeToFile(opts, resp, logger)
 
 	if bytes_written != response_size {
 		panic(fmt.Sprintf("wrote %d bytes out of %d bytes to %s", bytes_written, response_size, *opts.output))

runtime.R

@@ -23,26 +23,40 @@ cache_dir <- tempdir()
 na_if_null <- function(x) if (is.null(x)) "NULL" else x
 
 subset <- function(
-    id      = NULL,
-    comid   = NULL,
-    hl_uri  = NULL,
-    nldi    = NULL,
-    loc     = NULL,
-    layers  = c("divides", "nexus", "flowpaths", "network", "hydrolocations"),
+    id           = NULL,
+    comid        = NULL,
+    hl_uri       = NULL,
+    nldi_feature = NULL,
+    xy           = NULL,
+    layers = c("divides", "nexus", "flowpaths", "network", "hydrolocations"),
     version = c("pre-release", "v1.0", "v1.1", "v1.2")
 ) {
     version <- match.arg(version)
     s3_uri  <- paste0("s3://lynker-spatial/", version, "/")
 
+    missing_all <- is.null(id)     &&
+                   is.null(comid)  &&
+                   is.null(hl_uri) &&
+                   is.null(nldi_feature)   &&
+                   is.null(xy)
+
+    if (missing_all) {
+        return(list(
+            "response" = "Error",
+            "status"   = 400,
+            "message"  = "No ID parameters were given."
+        ))
+    }
+
     logger::log_info(glue::glue(
         "[subset] Received request:",
         "{{",
         "  s3: {s3_uri}, ",
         "  id: {na_if_null(id)}, ",
         "  comid: {na_if_null(comid)}, ",
         "  hl_uri: {na_if_null(hl_uri)}, ",
-        "  nldi: {na_if_null(nldi)}, ",
-        "  loc: {na_if_null(loc)}, ",
+        "  nldi_feature: {na_if_null(nldi_feature)}, ",
+        "  xy: {na_if_null(xy)}, ",
         "  layers: {layers}, ",
         "  version: {version}",
         "}}",
@@ -56,8 +70,8 @@ subset <- function(
         id           = id,
         comid        = comid,
         hl_uri       = hl_uri,
-        nldi_feature = nldi,
-        loc          = loc,
+        nldi_feature = nldi_feature,
+        xy           = xy,
         base_s3      = s3_uri,
         lyrs         = layers,
         outfile      = hf_tmp,
@@ -69,5 +83,5 @@ subset <- function(
 }
 
 lambdr::start_lambda(config = lambdr::lambda_config(
-    environ    = parent.frame()
+    environ = parent.frame()
 ))