diff --git a/.github/workflows/memory_test.yml b/.github/workflows/memory_test.yml deleted file mode 100644 index 6fb886b42..000000000 --- a/.github/workflows/memory_test.yml +++ /dev/null @@ -1,47 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: Memory Test - -on: - workflow_dispatch: - push: - branches: [ main ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ main ] - schedule: - - cron: '0 23 * * 5' # Every Sunday at 11 pm - -jobs: - memtest_sliderule: - name: Sliderule memory test - runs-on: ubuntu-latest - container: ghcr.io/icesat2-sliderule/sliderule-buildenv:latest - - steps: - - name: Install dependencies - run: | - apt-get update -y - apt-get install -y curl valgrind - - - name: Checkout sliderule repository - uses: actions/checkout@v3 - - - name: Build sliderule - run: | - make config-development - make - make install - chmod +x scripts/systests/memtest.sh - - - name: Run Valgrind - run: ./scripts/systests/memtest.sh diff --git a/.github/workflows/self_test.yml b/.github/workflows/self_test.yml index 497ece1cd..213ec4401 100644 --- a/.github/workflows/self_test.yml +++ b/.github/workflows/self_test.yml @@ -39,9 +39,11 @@ jobs: - name: Build sliderule run: | - make config-development + cd targets/slideruleearth-aws + make config-debug make - make install - name: Run test - run: sliderule ./scripts/selftests/test_runner.lua + run: | + cd targets/slideruleearth-aws + make selftest TEST=../../scripts/selftests/test_runner.lua diff --git a/Makefile b/Makefile index e5336f2ff..88737032e 100644 --- a/Makefile +++ b/Makefile @@ -1,241 +1,59 @@ ROOT = $(shell pwd) -BUILD ?= $(ROOT)/build -STAGE ?= $(ROOT)/stage -SLIDERULE_BUILD = $(BUILD)/sliderule -PGC_BUILD = $(BUILD)/pgc -ATLAS_BUILD = $(BUILD)/atlas -ICESAT2_BUILD = $(BUILD)/icesat2 -GEDI_BUILD = $(BUILD)/gedi -LANDSAT_BUILD = $(BUILD)/landsat -USGS3DEP_BUILD = $(BUILD)/usgs3dep -OPENDATA_BUILD = $(BUILD)/opendata -SWOT_BUILD = $(BUILD)/swot +BUILD = $(ROOT)/build ######################## -# Core Framework Targets +# Application Targets ######################## -default-build: ## default build of sliderule - make -j4 -C $(SLIDERULE_BUILD) +CFG = -DUSE_ARROW_PACKAGE=ON +CFG += -DUSE_AWS_PACKAGE=ON +CFG += -DUSE_CCSDS_PACKAGE=ON +CFG += -DUSE_GEO_PACKAGE=ON +CFG += -DUSE_H5_PACKAGE=ON +CFG += -DUSE_LEGACY_PACKAGE=ON +CFG += -DUSE_NETSVC_PACKAGE=ON +CFG += -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -all: default-build atlas pgc gedi icesat2 landsat usgs3dep opendata swot ## build everything +all: ## build code + make -j4 -C $(BUILD) -config: config-release ## configure make for default build +config: prep ## configure make for release version of sliderule + cd $(BUILD); cmake -DCMAKE_BUILD_TYPE=Release $(CFG) $(ROOT) -config-release: prep ## configure make for release version of sliderule binary - cd $(SLIDERULE_BUILD); cmake -DCMAKE_BUILD_TYPE=Release $(ROOT) - -config-debug: prep ## configure make for debug version of sliderule binary - cd $(SLIDERULE_BUILD); cmake -DCMAKE_BUILD_TYPE=Debug $(ROOT) - -DEVCFG = -DUSE_ARROW_PACKAGE=ON -DEVCFG += -DUSE_AWS_PACKAGE=ON -DEVCFG += -DUSE_CCSDS_PACKAGE=ON -DEVCFG += -DUSE_GEO_PACKAGE=ON -DEVCFG += -DUSE_H5_PACKAGE=ON -DEVCFG += -DUSE_LEGACY_PACKAGE=ON -DEVCFG += -DUSE_NETSVC_PACKAGE=ON -DEVCFG += -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - -config-development: prep ## configure make for development version of sliderule binary - cd $(SLIDERULE_BUILD); cmake -DCMAKE_BUILD_TYPE=Release $(DEVCFG) $(ROOT) - -config-development-debug: prep ## configure make for debug version of sliderule binary - cd $(SLIDERULE_BUILD); cmake -DCMAKE_BUILD_TYPE=Debug $(DEVCFG) -DENABLE_TRACING=ON $(ROOT) - -config-all: config-development config-atlas config-pgc config-gedi config-icesat2 config-landsat config-usgs3dep config-opendata config-swot ctags ## configure everything -config-all-debug: config-development-debug config-atlas-debug config-pgc-debug config-gedi-debug config-icesat2-debug config-landsat-debug config-usgs3dep-debug config-opendata-debug config-swot-debug ctags ## configure everything for debug +config-library: prep ## configure make for shared library libsliderule.so + cd $(BUILD); cmake -DCMAKE_BUILD_TYPE=Release -DSHARED_LIBRARY=ON $(CFG) $(ROOT) install: ## install sliderule to system - make -C $(SLIDERULE_BUILD) install - -install-all: install install-atlas install-pgc install-gedi install-icesat2 install-landsat install-usgs3dep install-opendata install-swot ## install everything + make -C $(BUILD) install uninstall: ## uninstall most recent install of sliderule from system - xargs rm < $(SLIDERULE_BUILD)/install_manifest.txt - -######################## -# Shared Library Targets -######################## - -config-library: prep ## configure make for shared library libsliderule.so - cd $(SLIDERULE_BUILD); cmake -DCMAKE_BUILD_TYPE=Release -DSHARED_LIBRARY=ON $(ROOT) - -##################### -# PGC Plugin Targets -##################### - -config-pgc-debug: prep ## configure make for pgc plugin - cd $(PGC_BUILD); cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/pgc - -config-pgc: prep ## configure make for pgc plugin - cd $(PGC_BUILD); cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/pgc - -pgc: ## build pgc plugin - make -j4 -C $(PGC_BUILD) - -install-pgc: ## install pgc plugin to system - make -C $(PGC_BUILD) install - -uninstall-pgc: ## uninstall most recent install of icesat2 plugin from system - xargs rm < $(PGC_BUILD)/install_manifest.txt - -######################## -# ATLAS Plugin Targets -######################## - -config-atlas: prep ## configure make for atlas plugin - cd $(ATLAS_BUILD); cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/atlas - -config-atlas-debug: prep ## configure make for atlas plugin - cd $(ATLAS_BUILD); cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/atlas - -atlas: ## build atlas plugin - make -j4 -C $(ATLAS_BUILD) - -install-atlas: ## install altas plugin to system - make -C $(ATLAS_BUILD) install - -uninstall-atlas: ## uninstall most recent install of atlas plugin from system - xargs rm < $(ATLAS_BUILD)/install_manifest.txt - -######################## -# ICESat-2 Plugin Targets -######################## - -config-icesat2: prep ## configure make for icesat2 plugin - cd $(ICESAT2_BUILD); cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/icesat2 - -config-icesat2-debug: prep ## configure make for icesat2 plugin - cd $(ICESAT2_BUILD); cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/icesat2 - -icesat2: ## build icesat2 plugin - make -j4 -C $(ICESAT2_BUILD) - -install-icesat2: ## install icesat2 plugin to system - make -C $(ICESAT2_BUILD) install - -uninstall-icesat2: ## uninstall most recent install of icesat2 plugin from system - xargs rm < $(ICESAT2_BUILD)/install_manifest.txt - -######################## -# GEDI Plugin Targets -######################## - -config-gedi: prep ## configure make for gedi plugin - cd $(GEDI_BUILD); cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/gedi - -config-gedi-debug: prep ## configure make for gedi plugin - cd $(GEDI_BUILD); cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/gedi - -gedi: ## build gedi plugin - make -j4 -C $(GEDI_BUILD) - -install-gedi: ## install gedi plugin to system - make -C $(GEDI_BUILD) install - -uninstall-gedi: ## uninstall most recent install of gedi plugin from system - xargs rm < $(GEDI_BUILD)/install_manifest.txt - -########################## -# Landsat Plugin Targets -########################## - -config-landsat-debug: prep ## configure make for landsat plugin - cd $(LANDSAT_BUILD); cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/landsat - -config-landsat: prep ## configure make for landsat plugin - cd $(LANDSAT_BUILD); cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/landsat - -landsat: ## build icesat2 plugin - make -j4 -C $(LANDSAT_BUILD) - -install-landsat: ## install icesat2 plugin to system - make -C $(LANDSAT_BUILD) install - -uninstall-landsat: ## uninstall most recent install of icesat2 plugin from system - xargs rm < $(LANDSAT_BUILD)/install_manifest.txt - -########################## -# USGS3DEP Plugin Targets -########################## - -config-usgs3dep-debug: prep ## configure make for usgs3dep plugin - cd $(USGS3DEP_BUILD); cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/usgs3dep - -config-usgs3dep: prep ## configure make for usgs3dep plugin - cd $(USGS3DEP_BUILD); cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/usgs3dep - -usgs3dep: ## build usgs3dep plugin - make -j4 -C $(USGS3DEP_BUILD) - -install-usgs3dep: ## install icesat2 plugin to system - make -C $(USGS3DEP_BUILD) install - -uninstall-usgs3dep: ## uninstall most recent install of icesat2 plugin from system - xargs rm < $(USGS3DEP_BUILD)/install_manifest.txt - -########################## -# OPENDATA Plugin Targets -########################## - -config-opendata-debug: prep ## configure make for opendata plugin - cd $(OPENDATA_BUILD); cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/opendata - -config-opendata: prep ## configure make for opendata plugin - cd $(OPENDATA_BUILD); cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/opendata - -opendata: ## build opendata plugin - make -j4 -C $(OPENDATA_BUILD) - -install-opendata: ## install icesat2 plugin to system - make -C $(OPENDATA_BUILD) install - -uninstall-opendata: ## uninstall most recent install of icesat2 plugin from system - xargs rm < $(OPENDATA_BUILD)/install_manifest.txt - -########################## -# SWOT Plugin Targets -########################## - -config-swot-debug: prep ## configure make for swot plugin - cd $(SWOT_BUILD); cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/swot - -config-swot: prep ## configure make for swot plugin - cd $(SWOT_BUILD); cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $(ROOT)/plugins/swot - -swot: ## build swot plugin - make -j4 -C $(SWOT_BUILD) - -install-swot: ## install icesat2 plugin to system - make -C $(SWOT_BUILD) install - -uninstall-swot: ## uninstall most recent install of icesat2 plugin from system - xargs rm < $(SWOT_BUILD)/install_manifest.txt + xargs rm < $(BUILD)/install_manifest.txt ######################## # Development Targets ######################## +TEST ?= + testmem: ## run memory test on sliderule - valgrind --leak-check=full --track-origins=yes --track-fds=yes sliderule $(testcase) + valgrind --leak-check=full --track-origins=yes --track-fds=yes sliderule $(TEST) testcpu: ## run cpu test on sliderule - valgrind --tool=callgrind sliderule $(testcase) + valgrind --tool=callgrind sliderule $(TEST) # kcachegrind callgrind.out. testheap: ## run heap test on sliderule - valgrind --tool=massif --time-unit=B --pages-as-heap=yes sliderule $(testcase) + valgrind --tool=massif --time-unit=B --pages-as-heap=yes sliderule $(TEST) # ms_print massif.out. testheaptrack: ## analyze results of heap test - # heaptrack sliderule $(testcase) + # heaptrack sliderule $(TEST) # heaptrack_gui heaptrack.sliderule..gz testcov: ## analyze results of test coverage report - lcov -c --directory $(SLIDERULE_BUILD) --output-file $(SLIDERULE_BUILD)/coverage.info - genhtml $(SLIDERULE_BUILD)/coverage.info --output-directory $(SLIDERULE_BUILD)/coverage_html - # firefox $(SLIDERULE_BUILD)/coverage_html/index.html + lcov -c --directory $(BUILD) --output-file $(BUILD)/coverage.info + genhtml $(BUILD)/coverage.info --output-directory $(BUILD)/coverage_html + # firefox $(BUILD)/coverage_html/index.html ######################## # Global Targets @@ -247,34 +65,13 @@ ctags: ## generate ctags /usr/bin/jq -s 'add' $(SLIDERULE_BUILD)/compile_commands.json $(PGC_BUILD)/compile_commands.json $(ICESAT2_BUILD)/compile_commands.json $(GEDI_BUILD)/compile_commands.json $(LANDSAT_BUILD)/compile_commands.json $(USGS3DEP_BUILD)/compile_commands.json $(OPENDATA_BUILD)/compile_commands.json $(SWOT_BUILD)/compile_commands.json > compile_commands.json prep: ## create necessary build directories - mkdir -p $(SLIDERULE_BUILD) - mkdir -p $(PGC_BUILD) - mkdir -p $(ATLAS_BUILD) - mkdir -p $(ICESAT2_BUILD) - mkdir -p $(GEDI_BUILD) - mkdir -p $(LANDSAT_BUILD) - mkdir -p $(USGS3DEP_BUILD) - mkdir -p $(OPENDATA_BUILD) - mkdir -p $(SWOT_BUILD) + mkdir -p $(BUILD) clean: ## clean last build - - make -C $(SLIDERULE_BUILD) clean - - make -C $(PGC_BUILD) clean - - make -C $(ATLAS_BUILD) clean - - make -C $(ICESAT2_BUILD) clean - - make -C $(GEDI_BUILD) clean - - make -C $(LANDSAT_BUILD) clean - - make -C $(USGS3DEP_BUILD) clean - - make -C $(OPENDATA_BUILD) clean - - make -C $(SWOT_BUILD) clean + - make -C $(BUILD) clean distclean: ## fully remove all non-version controlled files and directories - rm -Rf $(BUILD) - - rm -Rf $(STAGE) - - cd $(ROOT)/docs && ./clean.sh - - cd $(ROOT)/clients/python && ./clean.sh - - find -name ".cookies" -exec rm {} \; - - rm -f compile_commands.json help: ## that's me! @printf "\033[37m%-30s\033[0m %s\n" "#-----------------------------------------------------------------------------------------" diff --git a/README.md b/README.md index a8209f5bb..36bb784b9 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,6 @@ Options include: -DSHARED_LIBRARY=[ON|OFF] build sliderule as a shared library (overrides all other targets) default: OFF - -DENABLE_COMPAT=[ON|OFF] configure build for older tool chains (needed to build on CentOS 7) - default: OFF - -DENABLE_ADDRESS_SANITIZER=[ON|OFF] instrument code with for detecting memory errors default: OFF diff --git a/clients/python/tests/test_gedi.py b/clients/python/tests/test_gedi.py index 87d220784..20f9da1fe 100644 --- a/clients/python/tests/test_gedi.py +++ b/clients/python/tests/test_gedi.py @@ -79,7 +79,7 @@ def test_gedi(self, init): gdf = icesat2.atl06p(parms, resources=[resource]) assert init assert gdf.describe()["gedi.time"]["std"] == 0.0 - assert abs(gdf.describe()["gedi.value"]["mean"] - 3142.8526604140434) < 0.001 + assert abs(gdf.describe()["gedi.value"]["mean"] - 3143.5934365441703) < 0.001 assert gdf.describe()["gedi.file_id"]["max"] == 0.0 assert gdf.describe()["gedi.flags"]["max"] == 0.0 @@ -136,4 +136,4 @@ def test_gedi(self, init): assert key in gdf.keys() assert abs(gdf.describe()["canopy_openness"]["max"] - 10.390829086303711) < 0.001 df = gdf[gdf["gedi.value"] > -9999.0] - assert abs(sum(df["gedi.value"]) - 42551.55461392179) < 0.001 + assert abs(sum(df["gedi.value"]) - 42767.289459228516) < 0.001 diff --git a/packages/pistache/PistacheServer.cpp b/packages/pistache/PistacheServer.cpp index 685132418..80ed97437 100644 --- a/packages/pistache/PistacheServer.cpp +++ b/packages/pistache/PistacheServer.cpp @@ -103,7 +103,7 @@ PistacheServer::verb_t PistacheServer::str2verb (const char* str) const char* PistacheServer::sanitize (const char* filename) { const char* safe_filename = StringLib::replace(filename, PATH_DELIMETER_STR, "_"); - FString safe_pathname(0, "%s%c%s%c%s.lua", CONFDIR, PATH_DELIMETER, "api", PATH_DELIMETER, safe_filename); + FString safe_pathname("%s%c%s%c%s.lua", CONFDIR, PATH_DELIMETER, "api", PATH_DELIMETER, safe_filename); delete [] safe_filename; return safe_pathname.c_str(true); } diff --git a/plugins/gedi/CMakeLists.txt b/plugins/gedi/CMakeLists.txt index c8270d206..33d890827 100644 --- a/plugins/gedi/CMakeLists.txt +++ b/plugins/gedi/CMakeLists.txt @@ -12,6 +12,7 @@ add_library (gedi MODULE "") set_target_properties (gedi PROPERTIES OUTPUT_NAME gedi) set_target_properties (gedi PROPERTIES PREFIX "") +set_target_properties (gedi PROPERTIES CXX_STANDARD ${CXX_VERSION}) # Prerequisites # find_package (Lua "5.3") diff --git a/plugins/icesat2/CMakeLists.txt b/plugins/icesat2/CMakeLists.txt index d8e5861f8..d85987499 100644 --- a/plugins/icesat2/CMakeLists.txt +++ b/plugins/icesat2/CMakeLists.txt @@ -12,6 +12,7 @@ add_library (icesat2 MODULE "") set_target_properties (icesat2 PROPERTIES OUTPUT_NAME icesat2) set_target_properties (icesat2 PROPERTIES PREFIX "") +set_target_properties (icesat2 PROPERTIES CXX_STANDARD ${CXX_VERSION}) # Prerequisites # find_package (Lua "5.3") @@ -24,6 +25,7 @@ target_compile_definitions (icesat2 PUBLIC BUILDINFO="${BUILDINFO}") # Compile Definitions # target_compile_definitions (icesat2 PUBLIC BINID="${TGTVER}") +target_compile_options (icesat2 PUBLIC "$<$:-DWITH_UNITTESTS>") target_compile_options (icesat2 PUBLIC -Wall) # turn on "all" warnings target_compile_options (icesat2 PUBLIC -Wextra) # turn on "extra" warnings target_compile_options (icesat2 PUBLIC -Wreorder) # turn on warning for object initializer list order enforcement @@ -50,9 +52,8 @@ target_sources(icesat2 ${CMAKE_CURRENT_LIST_DIR}/plugin/GTDArray.cpp ${CMAKE_CURRENT_LIST_DIR}/plugin/Icesat2Parms.cpp ${CMAKE_CURRENT_LIST_DIR}/plugin/MeritRaster.cpp - ${CMAKE_CURRENT_LIST_DIR}/plugin/UT_Atl03Reader.cpp - ${CMAKE_CURRENT_LIST_DIR}/plugin/UT_Atl06Dispatch.cpp -) + $<$:${CMAKE_CURRENT_LIST_DIR}/plugin/UT_Atl03Reader.cpp> + $<$:${CMAKE_CURRENT_LIST_DIR}/plugin/UT_Atl06Dispatch.cpp>) # Include Directories # target_include_directories (icesat2 @@ -95,8 +96,8 @@ install ( ${CMAKE_CURRENT_LIST_DIR}/plugin/GTDArray.h ${CMAKE_CURRENT_LIST_DIR}/plugin/Icesat2Parms.h ${CMAKE_CURRENT_LIST_DIR}/plugin/MeritRaster.h - ${CMAKE_CURRENT_LIST_DIR}/plugin/UT_Atl03Reader.h - ${CMAKE_CURRENT_LIST_DIR}/plugin/UT_Atl06Dispatch.h + $<$:${CMAKE_CURRENT_LIST_DIR}/plugin/UT_Atl03Reader.h> + $<$:${CMAKE_CURRENT_LIST_DIR}/plugin/UT_Atl06Dispatch.h> DESTINATION ${INSTALLDIR}/include/sliderule/icesat2 ) diff --git a/plugins/icesat2/plugin/icesat2.cpp b/plugins/icesat2/plugin/icesat2.cpp index f5c59de1c..c19f229d4 100644 --- a/plugins/icesat2/plugin/icesat2.cpp +++ b/plugins/icesat2/plugin/icesat2.cpp @@ -68,8 +68,10 @@ int icesat2_open (lua_State *L) {"atl06", Atl06Dispatch::luaCreate}, {"atl06s", Atl06Reader::luaCreate}, {"atl08", Atl08Dispatch::luaCreate}, +#ifdef WITH_UNITTESTS {"ut_atl06", UT_Atl06Dispatch::luaCreate}, {"ut_atl03", UT_Atl03Reader::luaCreate}, +#endif {"version", icesat2_version}, {NULL, NULL} }; @@ -105,6 +107,12 @@ int icesat2_open (lua_State *L) LuaEngine::setAttrInt(L, "ATL08_TOP_OF_CANOPY", Icesat2Parms::ATL08_TOP_OF_CANOPY); LuaEngine::setAttrInt(L, "ATL08_UNCLASSIFIED", Icesat2Parms::ATL08_UNCLASSIFIED); + #ifdef WITH_UNITTESTS + LuaEngine::setAttrBool(L, "UNITTEST", true); + #else + LuaEngine::setAttrBool(L, "UNITTEST", false); + #endif + return 1; } diff --git a/plugins/icesat2/plugin/icesat2.h b/plugins/icesat2/plugin/icesat2.h index 067ffe649..76492a681 100644 --- a/plugins/icesat2/plugin/icesat2.h +++ b/plugins/icesat2/plugin/icesat2.h @@ -47,8 +47,11 @@ #include "GTDArray.h" #include "MeritRaster.h" #include "Icesat2Parms.h" + +#ifdef WITH_UNITTESTS #include "UT_Atl03Reader.h" #include "UT_Atl06Dispatch.h" +#endif /****************************************************************************** * PROTOTYPES diff --git a/plugins/icesat2/selftests/plugin_unittest.lua b/plugins/icesat2/selftests/plugin_unittest.lua index d6be2922b..f1adf4498 100644 --- a/plugins/icesat2/selftests/plugin_unittest.lua +++ b/plugins/icesat2/selftests/plugin_unittest.lua @@ -1,6 +1,9 @@ local runner = require("test_executive") local console = require("console") +-- Check If Present -- +if not icesat2.UNITTEST then return end + -- Setup -- atl06_dispatch = icesat2.ut_atl06() diff --git a/plugins/landsat/CMakeLists.txt b/plugins/landsat/CMakeLists.txt index 5e49aa49a..48a459bce 100644 --- a/plugins/landsat/CMakeLists.txt +++ b/plugins/landsat/CMakeLists.txt @@ -12,6 +12,7 @@ add_library (landsat MODULE "") set_target_properties (landsat PROPERTIES OUTPUT_NAME landsat) set_target_properties (landsat PROPERTIES PREFIX "") +set_target_properties (landsat PROPERTIES CXX_STANDARD ${CXX_VERSION}) # Prerequisites # find_package (Lua "5.3") diff --git a/plugins/opendata/CMakeLists.txt b/plugins/opendata/CMakeLists.txt index cbef12a3e..badab3de3 100644 --- a/plugins/opendata/CMakeLists.txt +++ b/plugins/opendata/CMakeLists.txt @@ -12,6 +12,7 @@ add_library (opendata MODULE "") set_target_properties (opendata PROPERTIES OUTPUT_NAME opendata) set_target_properties (opendata PROPERTIES PREFIX "") +set_target_properties (opendata PROPERTIES CXX_STANDARD ${CXX_VERSION}) # Prerequisites # find_package (Lua "5.3") diff --git a/plugins/pgc/CMakeLists.txt b/plugins/pgc/CMakeLists.txt index 2f5003a19..5cf06bd40 100644 --- a/plugins/pgc/CMakeLists.txt +++ b/plugins/pgc/CMakeLists.txt @@ -12,6 +12,7 @@ add_library (pgc MODULE "") set_target_properties (pgc PROPERTIES OUTPUT_NAME pgc) set_target_properties (pgc PROPERTIES PREFIX "") +set_target_properties (pgc PROPERTIES CXX_STANDARD ${CXX_VERSION}) # Prerequisites # find_package (Lua "5.3") diff --git a/plugins/swot/CMakeLists.txt b/plugins/swot/CMakeLists.txt index 0220d1cef..0e750e3fe 100644 --- a/plugins/swot/CMakeLists.txt +++ b/plugins/swot/CMakeLists.txt @@ -12,6 +12,7 @@ add_library (swot MODULE "") set_target_properties (swot PROPERTIES OUTPUT_NAME swot) set_target_properties (swot PROPERTIES PREFIX "") +set_target_properties (swot PROPERTIES CXX_STANDARD ${CXX_VERSION}) # Prerequisites # find_package (Lua "5.3") diff --git a/plugins/usgs3dep/CMakeLists.txt b/plugins/usgs3dep/CMakeLists.txt index 776a00a3f..2398ef28d 100644 --- a/plugins/usgs3dep/CMakeLists.txt +++ b/plugins/usgs3dep/CMakeLists.txt @@ -12,6 +12,7 @@ add_library (usgs3dep MODULE "") set_target_properties (usgs3dep PROPERTIES OUTPUT_NAME usgs3dep) set_target_properties (usgs3dep PROPERTIES PREFIX "") +set_target_properties (usgs3dep PROPERTIES CXX_STANDARD ${CXX_VERSION}) # Prerequisites # find_package (Lua "5.3") diff --git a/project-config.cmake b/project-config.cmake index de87b655c..787a7d8f4 100644 --- a/project-config.cmake +++ b/project-config.cmake @@ -80,7 +80,6 @@ option (SERVER_APP "Create sliderule server binary" ON) # Library Options # -option (ENABLE_COMPAT "Use C++11 for compatibility with older compilers" OFF) option (ENABLE_ADDRESS_SANITIZER "Instrument code with AddressSanitizer for memory error detection" OFF) option (ENABLE_TIME_HEARTBEAT "Instruct TimeLib to use a 1KHz heart beat timer to set millisecond time resolution" OFF) option (ENABLE_CUSTOM_ALLOCATOR "Override new and delete operators globally for debug purposes" OFF) @@ -111,11 +110,7 @@ string(REPLACE "v" "" LIBVER ${TGTVER}) # C++ Version # -if(${ENABLE_COMPAT}) - set(CXX_VERSION 11) -else() - set(CXX_VERSION 17) # required if using pistache package -endif() +set(CXX_VERSION 17) # required if using pistache package # Platform # diff --git a/scripts/selftests/test_runner.lua b/scripts/selftests/test_runner.lua index 3e664df51..912e09d4d 100644 --- a/scripts/selftests/test_runner.lua +++ b/scripts/selftests/test_runner.lua @@ -1,17 +1,8 @@ local runner = require("test_executive") -local console = require("console") -local global = require("global") local td = runner.rootdir(arg[0]) -- root directory -local loglvl = global.eval(arg[1]) or core.INFO +local incloud = arg[1] == "cloud" --- Initial Configuration -- --- console.monitor:config(core.LOG, loglvl) --- sys.setlvl(core.LOG, loglvl) - -local maxRuns = 1 -for runNum = 1, maxRuns do - - -- Run Core Self Tests -- +-- Run Core Self Tests -- if __core__ then runner.script(td .. "tcp_socket.lua") runner.script(td .. "udp_socket.lua") @@ -63,7 +54,7 @@ if __legacy__ then end -- Run ICESat-2 Plugin Self Tests -if __icesat2__ then +if __icesat2__ and incloud then local icesat2_td = td .. "../../plugins/icesat2/selftests/" runner.script(icesat2_td .. "plugin_unittest.lua") runner.script(icesat2_td .. "atl03_reader.lua") @@ -75,13 +66,13 @@ if __icesat2__ then end -- Run opendata Plugin Self Tests -if __opendata__ then +if __opendata__ and incloud then local opendata_td = td .. "../../plugins/opendata/selftests/" runner.script(opendata_td .. "worldcover_reader.lua") end -- Run PGC Plugin Self Tests -if __pgc__ then +if __pgc__ and incloud then local pgc_td = td .. "../../plugins/pgc/selftests/" runner.script(pgc_td .. "arcticdem_reader.lua") runner.script(pgc_td .. "temporal_filter_test.lua") @@ -95,25 +86,17 @@ if __pgc__ then end -- Run Landsat Plugin Self Tests -if __landsat__ then +if __landsat__ and incloud then local landsat_td = td .. "../../plugins/landsat/selftests/" runner.script(landsat_td .. "landsat_reader.lua") end -- Run usgs3dep Plugin Self Tests -if __usgs3dep__ then +if __usgs3dep__ and incloud then local usg2dep_td = td .. "../../plugins/usgs3dep/selftests/" runner.script(usg2dep_td .. "usgs3dep_reader.lua") end - -if maxRuns > 1 then - print(string.format("\n--------------------------------\nTest Repeat Run: %d of %d finished\n--------------------------------", runNum, maxRuns)) - sys.wait(3) -end - -end -- test repeat loop end - -- Report Results -- local errors = runner.report() diff --git a/targets/sigview-linux/Makefile b/targets/sigview-linux/Makefile new file mode 100644 index 000000000..09865c994 --- /dev/null +++ b/targets/sigview-linux/Makefile @@ -0,0 +1,74 @@ +# Makefile to build sliderule with atlas plugin + +ROOT = $(shell pwd)/../.. +BUILD = $(ROOT)/build +STAGE = $(ROOT)/stage + +SLIDERULE_BUILD_DIR = $(BUILD)/sliderule +SLIDERULE_STAGE_DIR = $(STAGE)/atlas +ATLAS_BUILD_DIR = $(BUILD)/altas + +CLANG_VER = "-12" +CLANG_OPT = -DCMAKE_USER_MAKE_RULES_OVERRIDE=$(ROOT)/platforms/linux/ClangOverrides.txt -D_CMAKE_TOOLCHAIN_PREFIX=llvm- +CLANG_CFG = export CC=clang$(CLANG_VER) && export CXX=clang++$(CLANG_VER) + +PKG_CFG += -DUSE_ARROW_PACKAGE=OFF +PKG_CFG += -DUSE_AWS_PACKAGE=OFF +PKG_CFG += -DUSE_CCSDS_PACKAGE=ON +PKG_CFG += -DUSE_GEO_PACKAGE=ON +PKG_CFG += -DUSE_H5_PACKAGE=ON +PKG_CFG += -DUSE_LEGACY_PACKAGE=ON +PKG_CFG += -DUSE_NETSVC_PACKAGE=ON +PKG_CFG += -DUSE_PISTACHE_PACKAGE=OFF + +DEBUG_CFG := -DCMAKE_BUILD_TYPE=Debug +DEBUG_CFG += -DINSTALLDIR=$(SLIDERULE_STAGE_DIR) +DEBUG_CFG += -DCMAKE_USER_MAKE_RULES_OVERRIDE=$(ROOT)/platforms/linux/ClangOverrides.txt -D_CMAKE_TOOLCHAIN_PREFIX=llvm- +DEBUG_CFG += -DENABLE_ADDRESS_SANITIZER=ON + +RELEASE_CFG := -DCMAKE_BUILD_TYPE=Release +RELEASE_CFG += -DINSTALLDIR=$(SLIDERULE_STAGE_DIR) + +######################## +# Build Targets +######################## + +all: atlas + +prep: ## create temporary directories needed for build + mkdir -p $(SLIDERULE_BUILD_DIR) + mkdir -p $(SLIDERULE_STAGE_DIR) + mkdir -p $(ATLAS_BUILD_DIR) + +config-debug: prep ## configure the server for running locally with debug symbols, optimizations off, static analysis, and address sanitizer + cd $(SLIDERULE_BUILD_DIR) && $(CLANG_CFG) && cmake $(PKG_CFG) $(DEBUG_CFG) $(ROOT) + cd $(ATLAS_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(ROOT)/plugins/atlas + +config-release: prep ## configure server to run a release version locally + cd $(SLIDERULE_BUILD_DIR) && cmake $(PKG_CFG) $(RELEASE_CFG) $(ROOT) + cd $(ATLAS_BUILD_DIR) && cmake $(RELEASE_CFG) $(ROOT)/plugins/atlas + +atlas: ## build the server using the local configuration + make -j4 -C $(SLIDERULE_BUILD_DIR) + make -C $(SLIDERULE_BUILD_DIR) install + make -j4 -C $(ATLAS_BUILD_DIR) + make -C $(ATLAS_BUILD_DIR) install + +run: ## run the server locally + $(SLIDERULE_STAGE_DIR)/bin/sliderule $(ROOT)/plugins/atlas/apps/server.lua $(ROOT)/plugins/atlas/config/config.json + +distclean: ## fully remove all non-version controlled files and directories + - rm -Rf $(BUILD) + - rm -Rf $(STAGE) + +######################## +# Help Target +######################## + +help: ## That's me! + @printf "\033[37m%-30s\033[0m %s\n" "#-----------------------------------------------------------------------------------------" + @printf "\033[37m%-30s\033[0m %s\n" "# Makefile Help |" + @printf "\033[37m%-30s\033[0m %s\n" "#-----------------------------------------------------------------------------------------" + @printf "\033[37m%-30s\033[0m %s\n" "#-target-----------------------description------------------------------------------------" + @grep -E '^[a-zA-Z_-].+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + @echo ENVVER:$(ENVVER) diff --git a/targets/slideruleearth-aws/Makefile b/targets/slideruleearth-aws/Makefile index 5463f8cd8..70ad848b8 100644 --- a/targets/slideruleearth-aws/Makefile +++ b/targets/slideruleearth-aws/Makefile @@ -1,6 +1,6 @@ # Notes: # -# (1) In order to complete the release process, the following must be installed and configured in your environment: +# In order to complete the release process, the following must be installed and configured in your environment: # * GitHub command line client (gh) # See https://github.com/cli/cli/blob/trunk/docs/install_linux.md for installation instructions. # The user must authenticate to GitHub via `gh auth login` @@ -15,24 +15,29 @@ # * Node.js # The javascript npm package for SlideRule is updated via a node.js script on release # -# (2) To release a version of SlideRule: +# To release a version of SlideRule: # 1. Update .aws/credentials file with a temporary session token; {profile} references your long term aws credentials, {username} is your aws account, {code} is your MFA token # $ aws --profile={profile} sts get-session-token --serial-number arn:aws:iam::742127912612:mfa/{username} --token-code={code} # 2. Login to AWS Elastic Container Registry # $ aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 742127912612.dkr.ecr.us-west-2.amazonaws.com -# 3. Login to GitHub Container Registry; {my_github_key} is a file storing an access key created for your account -# $ cat {my_github_key} | docker login ghcr.io -u jpswinski --password-stdin -# 4. Login to GitHub +# 3. Login to GitHub # $ gh auth login -# 5. Execute the makefile target to release the code; X.Y.Z is the version +# 4. Execute the makefile target to release the code; X.Y.Z is the version # $ make release VERSION=vX.Y.Z DOCKEROPTS=--no-cache # +# To update the build environment for GitHub actions +# 1. Login to GitHub Container Registry; {my_github_key} is a file storing an access key created for your account +# $ cat {my_github_key} | docker login ghcr.io -u jpswinski --password-stdin +# 2. Build the docker build environment on x86 machine (x86 is IMPORTANT to match GitHub actions environment) +# $ make sliderule-buildenv-docker +# 3. Push to GitHub container registry +# $ docker push ghcr.io/icesat2-sliderule/sliderule-buildenv:latest +# ROOT = $(shell pwd)/../.. BUILD = $(ROOT)/build STAGE = $(ROOT)/stage -SLIDERULE_SOURCE_DIR = $(ROOT) SLIDERULE_BUILD_DIR = $(BUILD)/sliderule TARGET_BUILD_DIR = $(BUILD)/target ICESAT2_BUILD_DIR = $(BUILD)/icesat2 @@ -53,6 +58,7 @@ PROXY_STAGE_DIR = $(STAGE)/proxy TF_STAGE_DIR = $(STAGE)/tf STATIC_WEB_SOURCE_DIR = $(ROOT)/docs STATIC_WEB_STAGE_DIR = $(STAGE)/website +INSTALL_DIR ?= $(SERVER_STAGE_DIR) VERSION ?= latest VERSION_TOKENS := $(subst ., ,$(lastword $(VERSION))) @@ -69,18 +75,13 @@ MYIP ?= $(shell (ip route get 1 | sed -n 's/^.*src \([0-9.]*\) .*$$/\1/p')) ENVVER = $(shell git --git-dir ../../.git --work-tree ../../.git describe --abbrev --dirty --always --tags --long) USERCFG ?= -TEST ?= $(SLIDERULE_SOURCE_DIR)/scripts/selftests/test_runner.lua -MOSAICS_PERFORMANCE_TEST ?= $(SLIDERULE_SOURCE_DIR)/plugins/pgc/systests/arcticdem_mosaic_perf.lua -STRIPS_PERFORMANCE_TEST ?= $(SLIDERULE_SOURCE_DIR)/plugins/pgc/systests/arcticdem_strips_perf.lua -SUBSET_PERFORMANCE_TEST ?= $(SLIDERULE_SOURCE_DIR)/scripts/systests/subset_perf_test.lua - CLANG_VER = "-12" -CLANG_OPT = -DCMAKE_USER_MAKE_RULES_OVERRIDE=$(SLIDERULE_SOURCE_DIR)/platforms/linux/ClangOverrides.txt -D_CMAKE_TOOLCHAIN_PREFIX=llvm- +CLANG_OPT = -DCMAKE_USER_MAKE_RULES_OVERRIDE=$(ROOT)/platforms/linux/ClangOverrides.txt -D_CMAKE_TOOLCHAIN_PREFIX=llvm- CLANG_CFG = export CC=clang$(CLANG_VER) && export CXX=clang++$(CLANG_VER) DEBUG_CFG := -DCMAKE_BUILD_TYPE=Debug -DEBUG_CFG += -DINSTALLDIR=$(SERVER_STAGE_DIR) -DEBUG_CFG += -DCMAKE_USER_MAKE_RULES_OVERRIDE=$(SLIDERULE_SOURCE_DIR)/platforms/linux/ClangOverrides.txt -D_CMAKE_TOOLCHAIN_PREFIX=llvm- +DEBUG_CFG += -DINSTALLDIR=$(INSTALL_DIR) +DEBUG_CFG += -DCMAKE_USER_MAKE_RULES_OVERRIDE=$(ROOT)/platforms/linux/ClangOverrides.txt -D_CMAKE_TOOLCHAIN_PREFIX=llvm- DEBUG_CFG += -DENABLE_ADDRESS_SANITIZER=ON DEBUG_CFG += -DUSE_ARROW_PACKAGE=ON DEBUG_CFG += -DUSE_AWS_PACKAGE=ON @@ -93,7 +94,7 @@ DEBUG_CFG += -DUSE_PISTACHE_PACKAGE=ON DEBUG_CFG += $(USERCFG) RELEASE_CFG := -DCMAKE_BUILD_TYPE=Release -RELEASE_CFG += -DINSTALLDIR=$(SERVER_STAGE_DIR) +RELEASE_CFG += -DINSTALLDIR=$(INSTALL_DIR) RELEASE_CFG += -DUSE_ARROW_PACKAGE=ON RELEASE_CFG += -DUSE_AWS_PACKAGE=ON RELEASE_CFG += -DUSE_GEO_PACKAGE=ON @@ -104,6 +105,10 @@ RELEASE_CFG += -DUSE_LEGACY_PACKAGE=OFF RELEASE_CFG += -DUSE_PISTACHE_PACKAGE=OFF RELEASE_CFG += $(USERCFG) +######################## +# Build Targets +######################## + all: sliderule prep: ## create temporary directories needed for build @@ -118,26 +123,26 @@ prep: ## create temporary directories needed for build mkdir -p $(SWOT_BUILD_DIR) config-debug: prep ## configure the server for running locally with debug symbols, optimizations off, static analysis, and address sanitizer - cd $(SLIDERULE_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) -DMAX_FREE_STACK_SIZE=1 -DENABLE_TRACING=ON $(SLIDERULE_SOURCE_DIR) - cd $(TARGET_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(SLIDERULE_SOURCE_DIR)/targets/slideruleearth-aws - cd $(ICESAT2_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/icesat2 - cd $(PGC_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/pgc - cd $(LANDSAT_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/landsat - cd $(GEDI_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/gedi - cd $(USGS3DEP_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/usgs3dep - cd $(OPENDATA_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/opendata - cd $(SWOT_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/swot + cd $(SLIDERULE_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) -DMAX_FREE_STACK_SIZE=1 -DENABLE_TRACING=ON $(ROOT) + cd $(TARGET_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(ROOT)/targets/slideruleearth-aws + cd $(ICESAT2_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(ROOT)/plugins/icesat2 + cd $(PGC_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(ROOT)/plugins/pgc + cd $(LANDSAT_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(ROOT)/plugins/landsat + cd $(GEDI_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(ROOT)/plugins/gedi + cd $(USGS3DEP_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(ROOT)/plugins/usgs3dep + cd $(OPENDATA_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(ROOT)/plugins/opendata + cd $(SWOT_BUILD_DIR) && $(CLANG_CFG) && cmake $(DEBUG_CFG) $(ROOT)/plugins/swot config-release: prep ## configure server to run a release version locally - cd $(SLIDERULE_BUILD_DIR) && cmake $(RELEASE_CFG) -DMAX_FREE_STACK_SIZE=1 $(SLIDERULE_SOURCE_DIR) - cd $(TARGET_BUILD_DIR) && cmake $(RELEASE_CFG) $(SLIDERULE_SOURCE_DIR)/targets/slideruleearth-aws - cd $(ICESAT2_BUILD_DIR) && cmake $(RELEASE_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/icesat2 - cd $(PGC_BUILD_DIR) && cmake $(RELEASE_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/pgc - cd $(LANDSAT_BUILD_DIR) && cmake $(RELEASE_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/landsat - cd $(GEDI_BUILD_DIR) && cmake $(RELEASE_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/gedi - cd $(USGS3DEP_BUILD_DIR) && cmake $(RELEASE_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/usgs3dep - cd $(OPENDATA_BUILD_DIR) && cmake $(RELEASE_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/opendata - cd $(SWOT_BUILD_DIR) && cmake $(RELEASE_CFG) $(SLIDERULE_SOURCE_DIR)/plugins/swot + cd $(SLIDERULE_BUILD_DIR) && cmake $(RELEASE_CFG) -DMAX_FREE_STACK_SIZE=1 $(ROOT) + cd $(TARGET_BUILD_DIR) && cmake $(RELEASE_CFG) $(ROOT)/targets/slideruleearth-aws + cd $(ICESAT2_BUILD_DIR) && cmake $(RELEASE_CFG) $(ROOT)/plugins/icesat2 + cd $(PGC_BUILD_DIR) && cmake $(RELEASE_CFG) $(ROOT)/plugins/pgc + cd $(LANDSAT_BUILD_DIR) && cmake $(RELEASE_CFG) $(ROOT)/plugins/landsat + cd $(GEDI_BUILD_DIR) && cmake $(RELEASE_CFG) $(ROOT)/plugins/gedi + cd $(USGS3DEP_BUILD_DIR) && cmake $(RELEASE_CFG) $(ROOT)/plugins/usgs3dep + cd $(OPENDATA_BUILD_DIR) && cmake $(RELEASE_CFG) $(ROOT)/plugins/opendata + cd $(SWOT_BUILD_DIR) && cmake $(RELEASE_CFG) $(ROOT)/plugins/swot sliderule: ## build the server using the local configuration make -j4 -C $(SLIDERULE_BUILD_DIR) @@ -160,19 +165,10 @@ sliderule: ## build the server using the local configuration make -C $(SWOT_BUILD_DIR) install run: ## run the server locally - IPV4=$(MYIP) ENVIRONMENT_VERSION=$(ENVVER) $(SERVER_STAGE_DIR)/bin/sliderule $(SLIDERULE_SOURCE_DIR)/scripts/apps/server.lua config.json - -selftest: ## run the self test on the server code - $(SERVER_STAGE_DIR)/bin/sliderule $(TEST) - -perfmtest: ## run mosaics performance test on the server code - $(SERVER_STAGE_DIR)/bin/sliderule $(MOSAICS_PERFORMANCE_TEST) + IPV4=$(MYIP) ENVIRONMENT_VERSION=$(ENVVER) $(INSTALL_DIR)/bin/sliderule $(ROOT)/scripts/apps/server.lua config.json -perfstest: ## run strips performance test on the server code - $(SERVER_STAGE_DIR)/bin/sliderule $(STRIPS_PERFORMANCE_TEST) - -perfsubsetest: ## run subset performance test - $(SERVER_STAGE_DIR)/bin/sliderule $(SUBSET_PERFORMANCE_TEST) +run-buildenv: ## run the build environment docker container + docker run -it -v $(ROOT):/host --rm --name buildenv $(ECR)/sliderule-buildenv:$(VERSION) manager-package: ## create the lambda python package for the manager cd terraform/manager/python && ./package.sh @@ -192,7 +188,7 @@ static-website-docker: ## make the static website docker image; needs VERSION mkdir -p $(STATIC_WEB_STAGE_DIR) cp -R $(STATIC_WEB_SOURCE_DIR)/rtd $(STATIC_WEB_STAGE_DIR) cp -R $(STATIC_WEB_SOURCE_DIR)/jekyll $(STATIC_WEB_STAGE_DIR) - rsync -a $(SLIDERULE_SOURCE_DIR) $(STATIC_WEB_STAGE_DIR)/sliderule --exclude build --exclude stage + rsync -a $(ROOT) $(STATIC_WEB_STAGE_DIR)/sliderule --exclude build --exclude stage -cp $(PYTHON_EXAMPLES_DIR)/examples/* $(STATIC_WEB_STAGE_DIR)/rtd/source/assets -cp $(PYTHON_EXAMPLES_DIR)/environment.yml $(STATIC_WEB_STAGE_DIR)/rtd/source/assets -cp -R /data/web $(STATIC_WEB_STAGE_DIR)/rtd/source/_static/html @@ -216,32 +212,22 @@ static-website-destroy: ## destroy static-website using terraform; needs DOMAIN cd terraform/static-website && terraform workspace select $(DOMAIN)-static-website || terraform workspace new $(DOMAIN)-static-website cd terraform/static-website && terraform destroy -var domain=$(DOMAIN) -var domain_root=$(DOMAIN_ROOT) -github-buildenv-docker: ## build sliderule build environment docker image for github actions - -rm -Rf $(BUILDENV_STAGE_DIR) - mkdir -p $(BUILDENV_STAGE_DIR) - cp docker/sliderule/Dockerfile.buildenv $(BUILDENV_STAGE_DIR)/Dockerfile - cd $(BUILDENV_STAGE_DIR); docker $(BUILDX) build $(DOCKEROPTS) -t $(GHCR)/sliderule-buildenv:latest $(DOCKER_PLATFORM) . - docker tag $(GHCR)/sliderule-buildenv:latest $(GHCR)/sliderule-buildenv:$(VERSION) - docker tag $(GHCR)/sliderule-buildenv:latest $(GHCR)/sliderule-buildenv:$(MAJOR_VERSION) - -github-buildenv-docker-push: ## push sliderule build environment docker image (must be x86 arch) to github container registry - docker push $(GHCR)/sliderule-buildenv:$(VERSION) - docker push $(GHCR)/sliderule-buildenv:$(MAJOR_VERSION) - -sliderule-buildenv-docker: ## build sliderule build environment docker image for github actions +sliderule-buildenv-docker: ## build sliderule build environment docker image -rm -Rf $(BUILDENV_STAGE_DIR) mkdir -p $(BUILDENV_STAGE_DIR) cp docker/sliderule/Dockerfile.buildenv $(BUILDENV_STAGE_DIR)/Dockerfile cd $(BUILDENV_STAGE_DIR); docker $(BUILDX) build $(DOCKEROPTS) -t $(ECR)/sliderule-buildenv:latest $(DOCKER_PLATFORM) . docker tag $(ECR)/sliderule-buildenv:latest $(ECR)/sliderule-buildenv:$(VERSION) docker tag $(ECR)/sliderule-buildenv:latest $(ECR)/sliderule-buildenv:$(MAJOR_VERSION) + docker tag $(ECR)/sliderule-buildenv:latest $(GHCR)/sliderule-buildenv:$(VERSION) + docker run -it -v ./docker/sliderule:/host --rm --name buildenv $(ECR)/sliderule-buildenv:$(VERSION) /bin/bash /host/libdep.sh sliderule-docker: ## build sliderule docker image using buildenv container; needs VERSION -rm -Rf $(SERVER_STAGE_DIR) - mkdir -p $(SERVER_STAGE_DIR)/etc - rsync -a $(SLIDERULE_SOURCE_DIR) $(SERVER_STAGE_DIR)/sliderule --exclude build --exclude stage + mkdir -p $(SERVER_STAGE_DIR) + rsync -a $(ROOT) $(SERVER_STAGE_DIR)/sliderule --exclude build --exclude stage cp docker/sliderule/Dockerfile.runtime $(SERVER_STAGE_DIR)/Dockerfile - cp docker/sliderule/*.* $(SERVER_STAGE_DIR)/etc + cp docker/sliderule/config.json $(SERVER_STAGE_DIR) cd $(SERVER_STAGE_DIR); docker $(BUILDX) build $(DOCKEROPTS) --build-arg repo=$(ECR) -t $(ECR)/sliderule:latest $(DOCKER_PLATFORM) . docker tag $(ECR)/sliderule:latest $(ECR)/sliderule:$(VERSION) docker tag $(ECR)/sliderule:latest $(ECR)/sliderule:$(MAJOR_VERSION) @@ -250,9 +236,9 @@ ilb-docker: ## build intelligent load balancer docker image; needs VERSION -rm -Rf $(ILB_STAGE_DIR) mkdir -p $(ILB_STAGE_DIR) cp docker/intelligent-load-balancer/* $(ILB_STAGE_DIR) - cp $(SLIDERULE_SOURCE_DIR)/targets/orchestrator-haproxy/orchestrator.lua $(ILB_STAGE_DIR) - cp $(SLIDERULE_SOURCE_DIR)/scripts/extensions/json.lua $(ILB_STAGE_DIR) - cp $(SLIDERULE_SOURCE_DIR)/scripts/extensions/prettyprint.lua $(ILB_STAGE_DIR) + cp $(ROOT)/targets/orchestrator-haproxy/orchestrator.lua $(ILB_STAGE_DIR) + cp $(ROOT)/scripts/extensions/json.lua $(ILB_STAGE_DIR) + cp $(ROOT)/scripts/extensions/prettyprint.lua $(ILB_STAGE_DIR) cd $(ILB_STAGE_DIR) && docker $(BUILDX) build $(DOCKEROPTS) -t $(ECR)/ilb:latest $(DOCKER_PLATFORM) . docker tag $(ECR)/ilb:latest $(ECR)/ilb:$(VERSION) docker tag $(ECR)/ilb:latest $(ECR)/ilb:$(MAJOR_VERSION) @@ -276,6 +262,7 @@ monitor-docker: ## build monitor docker image; needs VERSION docker tag $(ECR)/monitor:latest $(ECR)/monitor:$(MAJOR_VERSION) cluster-docker: ## build all cluster docker images + make sliderule-buildenv-docker make sliderule-docker make ilb-docker make proxy-docker @@ -330,6 +317,7 @@ release: ## release a version of sliderule; needs VERSION docker-clean: ## clean out old version of docker images; needs VERSION - docker image rm $(GHCR)/sliderule-buildenv:$(VERSION) + - docker image rm $(ECR)/sliderule-buildenv:$(VERSION) - docker image rm $(ECR)/sliderule:$(VERSION) - docker image rm $(ECR)/ilb:$(VERSION) - docker image rm $(ECR)/proxy:$(VERSION) @@ -337,11 +325,39 @@ docker-clean: ## clean out old version of docker images; needs VERSION - docker image rm $(ECR)/static-website:$(VERSION) - docker system prune -f -clean: ## remove build artifacts - make -C $(SLIDERULE_SOURCE_DIR) clean - distclean: ## fully remove all non-version controlled files and directories - make -C $(SLIDERULE_SOURCE_DIR) distclean + - rm -Rf $(BUILD) + - rm -Rf $(STAGE) + - cd $(ROOT)/docs && ./clean.sh + - cd $(ROOT)/clients/python && ./clean.sh + - find $(ROOT) -name ".cookies" -exec rm {} \; + - rm -f report.pdf + - rm -f report.xml + +######################## +# Test Targets +######################## + +TEST ?= $(ROOT)/scripts/selftests/test_runner.lua cloud +MOSAICS_PERFORMANCE_TEST ?= $(ROOT)/plugins/pgc/systests/arcticdem_mosaic_perf.lua +STRIPS_PERFORMANCE_TEST ?= $(ROOT)/plugins/pgc/systests/arcticdem_strips_perf.lua +SUBSET_PERFORMANCE_TEST ?= $(ROOT)/scripts/systests/subset_perf_test.lua + +selftest: ## run the self test on the server code + $(SERVER_STAGE_DIR)/bin/sliderule $(TEST) + +perfmtest: ## run mosaics performance test on the server code + $(SERVER_STAGE_DIR)/bin/sliderule $(MOSAICS_PERFORMANCE_TEST) + +perfstest: ## run strips performance test on the server code + $(SERVER_STAGE_DIR)/bin/sliderule $(STRIPS_PERFORMANCE_TEST) + +perfsubsetest: ## run subset performance test + $(SERVER_STAGE_DIR)/bin/sliderule $(SUBSET_PERFORMANCE_TEST) + +######################## +# Help Target +######################## help: ## That's me! @printf "\033[37m%-30s\033[0m %s\n" "#-----------------------------------------------------------------------------------------" diff --git a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv index ef19edca8..c1b1c39d1 100644 --- a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv +++ b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv @@ -9,6 +9,7 @@ RUN apt-get update && \ pkg-config \ cmake \ git \ + curl \ libreadline-dev \ liblua5.3-dev \ libcurl4-openssl-dev \ @@ -18,6 +19,11 @@ RUN apt-get update && \ libtiff-dev \ libsqlite3-dev \ sqlite3 \ + meson \ + llvm \ + clang-12 \ + clang-tidy \ + cppcheck \ && rm -rf /var/lib/apt/lists/* # install rapidjson dependency @@ -88,5 +94,14 @@ RUN make -j8 RUN make install RUN ldconfig +# install pistache dependency +WORKDIR / +RUN git clone https://github.com/pistacheio/pistache.git +WORKDIR /pistache +RUN meson setup build +RUN meson install -C build +RUN ldconfig + # support interactive mode +WORKDIR / CMD ["/bin/bash"] \ No newline at end of file diff --git a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime index af1c0254f..dcbcd2cd5 100644 --- a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime +++ b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime @@ -4,88 +4,12 @@ FROM $repo/sliderule-buildenv:latest AS buildenv # copy sliderule source repository COPY sliderule /sliderule -# build and install sliderule into staging -RUN mkdir -p /build/sliderule -WORKDIR /build/sliderule -RUN cmake -DCMAKE_BUILD_TYPE=Release \ - -DMAX_FREE_STACK_SIZE=1 \ - -DUSE_ARROW_PACKAGE=ON \ - -DUSE_AWS_PACKAGE=ON \ - -DUSE_H5_PACKAGE=ON \ - -DUSE_NETSVC_PACKAGE=ON \ - -DUSE_GEO_PACKAGE=ON \ - -DUSE_LEGACY_PACKAGE=OFF \ - -DUSE_CCSDS_PACKAGE=OFF \ - /sliderule -RUN make -j8 -RUN make install - -# build and install target configuration into staging -RUN mkdir -p /build/target -WORKDIR /build/target -RUN cmake -DCMAKE_BUILD_TYPE=Release \ - /sliderule/targets/slideruleearth-aws -RUN make -j8 -RUN make install - -# build and install gedi plugin into staging -RUN mkdir -p /build/gedi -WORKDIR /build/gedi -RUN cmake -DCMAKE_BUILD_TYPE=Release \ - /sliderule/plugins/gedi -RUN make -j8 -RUN make install - -# build and install icesat2 plugin into staging -RUN mkdir -p /build/icesat2 -WORKDIR /build/icesat2 -RUN cmake -DCMAKE_BUILD_TYPE=Release \ - /sliderule/plugins/icesat2 -RUN make -j8 -RUN make install - -# build and install landsat plugin into staging -RUN mkdir -p /build/landsat -WORKDIR /build/landsat -RUN cmake -DCMAKE_BUILD_TYPE=Release \ - /sliderule/plugins/landsat -RUN make -j8 -RUN make install - -# build and install opendata plugin into staging -RUN mkdir -p /build/opendata -WORKDIR /build/opendata -RUN cmake -DCMAKE_BUILD_TYPE=Release \ - /sliderule/plugins/opendata -RUN make -j8 -RUN make install - -# build and install pgc plugin into staging -RUN mkdir -p /build/pgc -WORKDIR /build/pgc -RUN cmake -DCMAKE_BUILD_TYPE=Release \ - /sliderule/plugins/pgc -RUN make -j8 -RUN make install - -# build and install swot plugin into staging -RUN mkdir -p /build/swot -WORKDIR /build/swot -RUN cmake -DCMAKE_BUILD_TYPE=Release \ - /sliderule/plugins/swot -RUN make -j8 -RUN make install - -# build and install usgs3dep plugin into staging -RUN mkdir -p /build/usgs3dep -WORKDIR /build/usgs3dep -RUN cmake -DCMAKE_BUILD_TYPE=Release \ - /sliderule/plugins/usgs3dep -RUN make -j8 -RUN make install +# build and install sliderule application +WORKDIR /sliderule/targets/slideruleearth-aws +RUN make config-release INSTALL_DIR=/usr/local && make # install project specific files -COPY etc/* /usr/local/etc/sliderule/ +COPY config.json /usr/local/etc/sliderule/ # install runtime dependencies FROM ubuntu:20.04 AS runtime diff --git a/targets/slideruleearth-aws/docker/sliderule/libdep.lock b/targets/slideruleearth-aws/docker/sliderule/libdep.lock new file mode 100644 index 000000000..6332ad54b --- /dev/null +++ b/targets/slideruleearth-aws/docker/sliderule/libdep.lock @@ -0,0 +1,8 @@ +PDAL: 2.6.0-9-g991388aa2 +PROJ: 9.0.0-1001-ge9c0e8c8 +arrow: apache-arrow-15.0.0.dev-45-g9569be83a +gdal: v3.8.0RC1-20-g170bc256a6 +geos: 3.12.0-61-g4118b3bb4 +libgeotiff: 1.7.1-39-gf2ffb6a +rapidjson: v1.1.0-744-gf9d53419 +pistache: 0.0.5-60-g1fb3465 diff --git a/targets/slideruleearth-aws/docker/sliderule/libdep.sh b/targets/slideruleearth-aws/docker/sliderule/libdep.sh new file mode 100755 index 000000000..d4823c600 --- /dev/null +++ b/targets/slideruleearth-aws/docker/sliderule/libdep.sh @@ -0,0 +1,8 @@ +echo PDAL: $(git --git-dir /PDAL/.git --work-tree /PDAL describe --abbrev --dirty --always --tags --long) > /host/libdep.lock +echo PROJ: $(git --git-dir /PROJ/.git --work-tree /PROJ describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock +echo arrow: $(git --git-dir /arrow/.git --work-tree /arrow describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock +echo gdal: $(git --git-dir /gdal/.git --work-tree /gdal describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock +echo geos: $(git --git-dir /geos/.git --work-tree /geos describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock +echo libgeotiff: $(git --git-dir /libgeotiff/.git --work-tree /libgeotiff describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock +echo rapidjson: $(git --git-dir /rapidjson/.git --work-tree /rapidjson describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock +echo pistache: $(git --git-dir /pistache/.git --work-tree /pistache describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock \ No newline at end of file