atl06 subsetting implemented

clients/python/sliderule/earthdata.py

@@ -90,6 +90,8 @@
     "usgs3dep-1meter-dem": "Digital Elevation Model (DEM) 1 meter",
     "landsat-hls": "HLS",
     "icesat2": "ATL03",
+    "icesat2-atl06": "ATL06",
+    "icesat2-atl08": "ATL08",
     "atlas-local": "ATL03",
     "nsidc-s3": "ATL03"
 }

clients/python/sliderule/gedi.py

@@ -156,16 +156,17 @@ def __processing_request(parm, asset, callbacks, resources, keep_id, as_numpy_ar
         tstart = time.perf_counter()
 
         # Default the Asset
-        if "asset" not in parm:
-            parm["asset"] = asset
+        rqst_parm = parm.copy()
+        if "asset" not in rqst_parm:
+            rqst_parm["asset"] = asset
 
         # Get List of Resources from CMR (if not supplied)
-        resources = earthdata.search(parm, resources)
+        resources = earthdata.search(rqst_parm, resources)
 
         # Build GEDI Request
         rqst = {
             "resources": resources,
-            "parms": parm
+            "parms": rqst_parm
         }
 
         # Make API Processing Request

clients/python/sliderule/icesat2.py

@@ -260,19 +260,20 @@ def __flattenbatches(rsps, rectype, batch_column, parm, keep_id, as_numpy_array,
 #
 # Build Request
 #
-def __build_request(parm, resources):
+def __build_request(parm, resources, default_asset='icesat2'):
 
     # Default the Asset
-    if "asset" not in parm:
-        parm["asset"] = "icesat2"
+    rqst_parm = parm.copy()
+    if "asset" not in rqst_parm:
+        rqst_parm["asset"] = default_asset
 
     # Get List of Resources
-    resources = earthdata.search(parm, resources)
+    resources = earthdata.search(rqst_parm, resources)
 
     # Build Request
     return {
         "resources": resources,
-        "parms": parm
+        "parms": rqst_parm
     }
 
 
@@ -463,7 +464,7 @@ def atl06sp(parm, callbacks={}, resources=None, keep_id=False, as_numpy_array=Fa
         tstart = time.perf_counter()
 
         # Build Request
-        rqst = __build_request(parm, resources)
+        rqst = __build_request(parm, resources, default_asset="icesat2-atl06")
 
         # Make API Processing Request
         rsps = sliderule.source("atl06sp", rqst, stream=True, callbacks=callbacks)

clients/python/sliderule/swot.py

@@ -130,16 +130,17 @@ def swotl2p(parm, callbacks={}, resources=None):
         tstart = time.perf_counter()
 
         # Default the Asset
-        if "asset" not in parm:
-            parm["asset"] = "swot-sim-ecco-llc4320"
+        rqst_parm = parm.copy()
+        if "asset" not in rqst_parm:
+            rqst_parm["asset"] = "swot-sim-ecco-llc4320"
 
         # Get List of Resources from CMR (if not supplied)
-        resources = earthdata.search(parm, resources)
+        resources = earthdata.search(rqst_parm, resources)
 
         # Build GEDI Request
         rqst = {
             "resources": resources,
-            "parms": parm
+            "parms": rqst_parm
         }
 
         # Make API Processing Request

plugins/icesat2/README.md

@@ -43,7 +43,7 @@ This plugin supplies the following record types:
 * `waverec`: return waveform built from ATL03 photons
 
 The plugin supplies the following lua user data types:
-* `icesat2.atl03(<url>, <outq_name>, [<parms>])`: ATL03 reader base object
+* `icesat2.atl03s(<url>, <outq_name>, [<parms>])`: ATL03 reader base object
 * `icesat2.atl03indexer(<asset>, <resource table>, <outq_name>, [<num threads>])`: ATL03 indexer base object
 * `icesat2.atl06(<outq name>)`: ATL06 dispatch object
 * `icesat2.atl08(<outq name>)`: ATL08 dispatch object

plugins/icesat2/endpoints/atl03s.lua

@@ -25,6 +25,6 @@ local rqst_parms    = icesat2.parms(parms)
 local proc          = georesource.initialize(resource, parms, nil, args)
 
 if proc then
-    local reader    = icesat2.atl03(proc.asset, resource, args.result_q, rqst_parms, false)
+    local reader    = icesat2.atl03s(proc.asset, resource, args.result_q, rqst_parms, false)
     local status    = georesource.waiton(resource, parms, nil, reader, nil, proc.sampler_disp, proc.userlog, true)
 end

plugins/icesat2/endpoints/atl06.lua

@@ -27,6 +27,6 @@ local algo          = icesat2.atl06(args.result_q, rqst_parms)
 local proc          = georesource.initialize(resource, parms, algo, args)
 
 if proc then
-    local reader    = icesat2.atl03(proc.asset, resource, proc.source_q, rqst_parms, true)
+    local reader    = icesat2.atl03s(proc.asset, resource, proc.source_q, rqst_parms, true)
     local status    = georesource.waiton(resource, parms, algo, reader, proc.algo_disp, proc.sampler_disp, proc.userlog, true)
 end

plugins/icesat2/endpoints/atl06s.lua

@@ -25,6 +25,6 @@ local rqst_parms    = icesat2.parms(parms)
 local proc          = georesource.initialize(resource, parms, nil, args)
 
 if proc then
-    local reader    = icesat2.atl03(proc.asset, resource, args.result_q, rqst_parms, false)
+    local reader    = icesat2.atl06s(proc.asset, resource, args.result_q, rqst_parms, false)
     local status    = georesource.waiton(resource, parms, nil, reader, nil, proc.sampler_disp, proc.userlog, true)
 end

plugins/icesat2/endpoints/atl08.lua

@@ -27,6 +27,6 @@ local algo          = icesat2.atl08(args.result_q, rqst_parms)
 local proc          = georesource.initialize(resource, parms, algo, args)
 
 if proc then
-    local reader    = icesat2.atl03(proc.asset, resource, proc.source_q, rqst_parms, true)
+    local reader    = icesat2.atl03s(proc.asset, resource, proc.source_q, rqst_parms, true)
     local status    = georesource.waiton(resource, parms, algo, reader, proc.algo_disp, proc.sampler_disp, proc.userlog, false)
 end

plugins/icesat2/plugin/Atl06Reader.cpp

@@ -49,6 +49,8 @@
 
 const char* Atl06Reader::elRecType = "atl06srec.elevation";
 const RecordObject::fieldDef_t Atl06Reader::elRecDef[] = {
+    {"extent_id",               RecordObject::UINT64,   offsetof(elevation_t, extent_id),               1,  NULL, NATIVE_FLAGS},
+// land_ice_segments
     {"time",                    RecordObject::TIME8,    offsetof(elevation_t, time_ns),                 1,  NULL, NATIVE_FLAGS},
     {"h_li",                    RecordObject::FLOAT,    offsetof(elevation_t, h_li),                    1,  NULL, NATIVE_FLAGS},
     {"h_li_sigma",              RecordObject::FLOAT,    offsetof(elevation_t, h_li_sigma),              1,  NULL, NATIVE_FLAGS},

plugins/icesat2/plugin/icesat2.cpp

@@ -63,7 +63,7 @@ int icesat2_open (lua_State *L)
 {
     static const struct luaL_Reg icesat2_functions[] = {
         {"parms",               Icesat2Parms::luaCreate},
-        {"atl03",               Atl03Reader::luaCreate},
+        {"atl03s",              Atl03Reader::luaCreate},
         {"atl03indexer",        Atl03Indexer::luaCreate},
         {"atl06",               Atl06Dispatch::luaCreate},
         {"atl06s",              Atl06Reader::luaCreate},

plugins/icesat2/selftests/atl03_ancillary.lua

@@ -25,7 +25,7 @@ end
 
 local cnt = 0
 local recq = msg.subscribe("atl03-ancillary-recq")
-local f = icesat2.atl03(nsidc_s3, "ATL03_20181017222812_02950102_005_01.h5", "atl03-ancillary-recq", icesat2.parms({cnf=4, track=icesat2.RPT_1, atl03_geo_fields={"solar_elevation"}}))
+local f = icesat2.atl03s(nsidc_s3, "ATL03_20181017222812_02950102_005_01.h5", "atl03-ancillary-recq", icesat2.parms({cnf=4, track=icesat2.RPT_1, atl03_geo_fields={"solar_elevation"}}))
 while true do
     local rec = recq:recvrecord(15000)
     if rec == nil then

plugins/icesat2/selftests/atl03_reader.lua

@@ -22,7 +22,7 @@ end
 
 print('\n------------------\nTest01: Atl03 Reader \n------------------')
 
-local f1_0 = icesat2.atl03(nsidc_s3, "missing_file", "tmpq", icesat2.parms({srt=icesat2.SRT_SEA_ICE, cnf=icesat2.CNF_NOT_CONSIDERED, track=icesat2.RPT_1}))
+local f1_0 = icesat2.atl03s(nsidc_s3, "missing_file", "tmpq", icesat2.parms({srt=icesat2.SRT_SEA_ICE, cnf=icesat2.CNF_NOT_CONSIDERED, track=icesat2.RPT_1}))
 local p1_0 = f1_0:parms()
 
 runner.check(p1_0.srt == icesat2.SRT_SEA_ICE, "Failed to set surface type")
@@ -34,7 +34,7 @@ runner.check(p1_0.cnf[icesat2.CNF_SURFACE_LOW], "Failed to set _low_")
 runner.check(p1_0.cnf[icesat2.CNF_SURFACE_MEDIUM], "Failed to set _medium_")
 runner.check(p1_0.cnf[icesat2.CNF_SURFACE_HIGH], "Failed to set _high_")
 
-local f1_1 = icesat2.atl03(nsidc_s3, "missing_file", "tmpq", icesat2.parms({srt=icesat2.SRT_SEA_ICE, cnf={icesat2.CNF_NOT_CONSIDERED}, track=icesat2.RPT_1}))
+local f1_1 = icesat2.atl03s(nsidc_s3, "missing_file", "tmpq", icesat2.parms({srt=icesat2.SRT_SEA_ICE, cnf={icesat2.CNF_NOT_CONSIDERED}, track=icesat2.RPT_1}))
 local p1_1 = f1_1:parms()
 
 runner.check(not p1_1.cnf[icesat2.CNF_POSSIBLE_TEP], "Failed to not set _tep_")
@@ -45,7 +45,7 @@ runner.check(not p1_1.cnf[icesat2.CNF_SURFACE_LOW], "Failed to not set _low_")
 runner.check(not p1_1.cnf[icesat2.CNF_SURFACE_MEDIUM], "Failed to not set _medium_")
 runner.check(not p1_1.cnf[icesat2.CNF_SURFACE_HIGH], "Failed to not set _high_")
 
-local f1_2 = icesat2.atl03(nsidc_s3, "missing_file", "tmpq", icesat2.parms({srt=icesat2.SRT_SEA_ICE, cnf={"atl03_not_considered"}, track=icesat2.RPT_1}))
+local f1_2 = icesat2.atl03s(nsidc_s3, "missing_file", "tmpq", icesat2.parms({srt=icesat2.SRT_SEA_ICE, cnf={"atl03_not_considered"}, track=icesat2.RPT_1}))
 local p1_2 = f1_2:parms()
 
 runner.check(not p1_2.cnf[icesat2.CNF_POSSIBLE_TEP], "Failed to not set _tep_")
@@ -56,7 +56,7 @@ runner.check(not p1_2.cnf[icesat2.CNF_SURFACE_LOW], "Failed to not set _low_")
 runner.check(not p1_2.cnf[icesat2.CNF_SURFACE_MEDIUM], "Failed to not set _medium_")
 runner.check(not p1_2.cnf[icesat2.CNF_SURFACE_HIGH], "Failed to not set _high_")
 
-local f1_3 = icesat2.atl03(nsidc_s3, "missing_file", "tmpq",icesat2.parms({srt=icesat2.SRT_SEA_ICE, cnf={"atl03_low", "atl03_medium", "atl03_high"}, track=icesat2.RPT_1}))
+local f1_3 = icesat2.atl03s(nsidc_s3, "missing_file", "tmpq",icesat2.parms({srt=icesat2.SRT_SEA_ICE, cnf={"atl03_low", "atl03_medium", "atl03_high"}, track=icesat2.RPT_1}))
 local p1_3 = f1_3:parms()
 
 runner.check(not p1_3.cnf[icesat2.CNF_POSSIBLE_TEP], "Failed to not set _tep_")
@@ -71,7 +71,7 @@ print('\n------------------\nTest02: Atl03 Extent Record\n------------------')
 
 local recq = msg.subscribe("atl03-reader-recq")
 local tstart = time.latch()
-local f2 = icesat2.atl03(nsidc_s3, "ATL03_20200304065203_10470605_005_01.h5", "atl03-reader-recq", icesat2.parms({cnf=4, track=icesat2.RPT_1}))
+local f2 = icesat2.atl03s(nsidc_s3, "ATL03_20200304065203_10470605_005_01.h5", "atl03-reader-recq", icesat2.parms({cnf=4, track=icesat2.RPT_1}))
 local extentrec = recq:recvrecord(15000)
 print("Time to execute: "..tostring(time.latch() - tstart))
 

plugins/icesat2/selftests/atl06_ancillary.lua

@@ -30,7 +30,7 @@ local algo = icesat2.atl06("atl06-ancillary-resultq", parms)
 local algo_disp = core.dispatcher("atl06-ancillary-recq")
 algo_disp:attach(algo, "atl03rec")
 algo_disp:run()
-local reader = icesat2.atl03(nsidc_s3, "ATL03_20181017222812_02950102_005_01.h5", "atl06-ancillary-recq", parms)
+local reader = icesat2.atl03s(nsidc_s3, "ATL03_20181017222812_02950102_005_01.h5", "atl06-ancillary-recq", parms)
 
 while true do
     local rec = resultq:recvrecord(15000)

targets/server-linux/CMakeLists.txt

@@ -10,4 +10,3 @@ set_target_properties (sliderule PROPERTIES CXX_STANDARD ${CXX_VERSION})
 target_link_libraries (sliderule PUBLIC slideruleLib)
 
 install (TARGETS sliderule DESTINATION ${INSTALLDIR}/bin)
-install (FILES ${CMAKE_CURRENT_LIST_DIR}/asset_directory.csv DESTINATION ${CONFDIR})

targets/slideruleearth-aws/CMakeLists.txt

@@ -0,0 +1,6 @@
+project (CLOUD LANGUAGES CXX)
+cmake_minimum_required (VERSION 3.13.0) # The minimum CMake version is chosen to enable policy CMP0079
+include(${CMAKE_SOURCE_DIR}/../../project-config.cmake)
+
+message (STATUS "Configuring cloud deployment")
+install (FILES ${CMAKE_CURRENT_LIST_DIR}/asset_directory.csv DESTINATION ${CONFDIR})

targets/slideruleearth-aws/Makefile

@@ -34,6 +34,7 @@ STAGE = $(ROOT)/stage
 
 SLIDERULE_SOURCE_DIR = $(ROOT)
 SLIDERULE_BUILD_DIR = $(BUILD)/sliderule
+TARGET_BUILD_DIR = $(BUILD)/target
 ICESAT2_BUILD_DIR = $(BUILD)/icesat2
 PGC_BUILD_DIR = $(BUILD)/pgc
 LANDSAT_BUILD_DIR = $(BUILD)/landsat
@@ -107,6 +108,7 @@ all: sliderule
 
 prep: ## create temporary directories needed for build
 	mkdir -p $(SLIDERULE_BUILD_DIR)
+	mkdir -p $(TARGET_BUILD_DIR)
 	mkdir -p $(ICESAT2_BUILD_DIR)
 	mkdir -p $(PGC_BUILD_DIR)
 	mkdir -p $(LANDSAT_BUILD_DIR)
@@ -117,6 +119,7 @@ prep: ## create temporary directories needed for build
 
 config-debug: prep ## configure the server for running locally with debug symbols, optimizations off, static analysis, and address sanitizer
 	cd $(SLIDERULE_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) -DMAX_FREE_STACK_SIZE=1 -DENABLE_TRACING=ON $(SLIDERULE_SOURCE_DIR)
+	cd $(TARGET_BUILD_DIR)    && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(SLIDERULE_SOURCE_DIR)/targets/slideruleearth-aws
 	cd $(ICESAT2_BUILD_DIR)   && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/icesat2
 	cd $(PGC_BUILD_DIR)       && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/pgc
 	cd $(LANDSAT_BUILD_DIR)   && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/landsat
@@ -127,6 +130,7 @@ config-debug: prep ## configure the server for running locally with debug symbol
 
 config-release: prep ## configure server to run a release version locally
 	cd $(SLIDERULE_BUILD_DIR) && cmake $(RELEASE_CFG) -DMAX_FREE_STACK_SIZE=1 $(SLIDERULE_SOURCE_DIR)
+	cd $(TARGET_BUILD_DIR)    && cmake $(RELEASE_CFG) $(SLIDERULE_SOURCE_DIR)/targets/slideruleearth-aws
 	cd $(ICESAT2_BUILD_DIR)   && cmake $(RELEASE_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/icesat2
 	cd $(PGC_BUILD_DIR)       && cmake $(RELEASE_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/pgc
 	cd $(LANDSAT_BUILD_DIR)   && cmake $(RELEASE_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/landsat
@@ -138,6 +142,8 @@ config-release: prep ## configure server to run a release version locally
 sliderule: ## build the server using the local configuration
 	make -j4 -C $(SLIDERULE_BUILD_DIR)
 	make -C $(SLIDERULE_BUILD_DIR) install
+	make -j4 -C $(TARGET_BUILD_DIR)
+	make -C $(TARGET_BUILD_DIR) install
 	make -j4 -C $(ICESAT2_BUILD_DIR)
 	make -C $(ICESAT2_BUILD_DIR) install
 	make -j4 -C $(PGC_BUILD_DIR)

targets/server-linux/asset_directory.csv → targets/slideruleearth-aws/asset_directory.csv

@@ -1,5 +1,7 @@
 asset,                      identity,       driver,     path,                                                                   index,              region,        endpoint
 icesat2,                    nsidc-cloud,    cumulus,    nsidc-cumulus-prod-protected,                                           nil,                us-west-2,     https://s3.us-west-2.amazonaws.com
+icesat2-atl06,              nsidc-cloud,    cumulus,    nsidc-cumulus-prod-protected,                                           nil,                us-west-2,     https://s3.us-west-2.amazonaws.com
+icesat2-atl08,              nsidc-cloud,    cumulus,    nsidc-cumulus-prod-protected,                                           nil,                us-west-2,     https://s3.us-west-2.amazonaws.com
 gedil4a,                    ornl-cloud,     s3,         ornl-cumulus-prod-protected/gedi/GEDI_L4A_AGB_Density_V2_1/data,        nil,                us-west-2,     https://s3.us-west-2.amazonaws.com
 gedil4b,                    ornl-cloud,     s3,         /vsis3/ornl-cumulus-prod-protected/gedi/GEDI_L4B_Gridded_Biomass_V2_1/data,    GEDI04_B_MW019MW223_02_002_02_R01000M_V2.tif,              us-west-2, https://s3.us-west-2.amazonaws.com
 gedil3-elevation,           ornl-cloud,     s3,         /vsis3/ornl-cumulus-prod-protected/gedi/GEDI_L3_LandSurface_Metrics_V2/data,   GEDI03_elev_lowestmode_mean_2019108_2022019_002_03.tif,    us-west-2, https://s3.us-west-2.amazonaws.com

targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime

@@ -20,6 +20,14 @@ RUN cmake -DCMAKE_BUILD_TYPE=Release \
 RUN make -j8
 RUN make install
 
+# build and install target configuration into staging
+RUN mkdir -p /build/target
+WORKDIR /build/target
+RUN cmake -DCMAKE_BUILD_TYPE=Release \
+          /sliderule/targets/slideruleearth-aws
+RUN make -j8
+RUN make install
+
 # build and install gedi plugin into staging
 RUN mkdir -p /build/gedi
 WORKDIR /build/gedi