diff --git a/.gitignore b/.gitignore index 93f7b58..14c636b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,8 @@ src/config.ini src/cmd.txt src/auth.json -src/utils/hide_copyright_info.ipynb +src/query/hide_copyright_info.ipynb +data/for_arcgis_dashboard/* ## VS CODE .vscode diff --git a/README.md b/README.md index 443e6ff..f75945b 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,15 @@ **Perceptual model** is defined as: > An expert summary of the watershed and its runoff processes often supported by field observations. Perceptual models are often presented as a schematic figure, although such a figure will necessarily simplify the hydrologist's complex mental model (McMillan et al., 2022) -This repo contains a released version of the perceptual data model database from https://doi.org/10.1002/hyp.14845. Currently our database holds **400 models** in both text (269) and figure (131) format collected from hydrologic literature. +This repo contains a released version of the perceptual data model database, used for the manuscript under review **"Global patterns in observed hydrologic processes (McMillan et al.)"**. Currently our database holds **400 models** in the form of both text (269) and figure (131, including 63 identified in [McMillan et al., 2022](https://doi.org/10.1002/hyp.14845)) collected from hydrologic literature. Visit **[the perceptual model interactive map](http://www.mcmillanhydrology.org/PerceptualModelDashboard.html)** for the visualization :world_map: ## Installation/Getting Started +If you are only interested in the pre-built database and not the building process, you can download the database backup (SQLdump) from ```data/backup/*.backup```. Alternatively, check csv files under ```data``` directory. + +The following explains the code used to build the Postgres database and webmap. + **1. Create your environment** Use Conda to create an environment ``` @@ -21,17 +25,18 @@ Run the following code in order: - `0-debug_excelsheets.ipynb` - `1-build_database.ipynb` +### Query database +`src/query` contains example scripts that can be used to query the database: +- `calc_stats.sql` holds query scripts to calculate statistics used in [the paper McMillan et al., 2022](https://doi.org/10.1002/hyp.14845) + - Note that analysis for the manuscript "Global patterns in observed hydrologic processes (McMillan et al.)" was implemented using ArcGIS software +- Use `debug_built_database.sql` to debug the database if the SQL database or code (`1-build_database.ipynb`) are not working as expected + ### Create webmap `src/webmap` contains example scripts that are used to create [the ArcGIS interactive webmap](http://www.mcmillanhydrology.org/PerceptualModelDashboard.html). To initiate the webmap after building the SQL database: - Run `init_create_webmap.ipynb` to initiate a webmap - Or, run `update_webmap.ipynb` to update an existing webmap -### Other utilities -`src/utils` contains other utilities: -- `calc_stats.sql` holds query scripts to calculate statistics used in [the paper](https://doi.org/10.1002/hyp.14845) -- Use `debug_built_database.sql` to debug the database if the SQL database or code (`1-build_database.ipynb`) are not working as expected - ## Resources - [The database design (ER diagram)](https://dbdiagram.io/d/63f6895b296d97641d830705) - [The hydrologic process taxonomy used in the analysis](http://mcmillanhydrology.org/ProcessTaxonomy/ProcessTaxonomyDiagram.html) diff --git a/data/backup/20240717.backup b/data/backup/20240717.backup new file mode 100644 index 0000000..858d9bb Binary files /dev/null and b/data/backup/20240717.backup differ diff --git a/src/utils/calc_stats.sql b/src/query/calc_stats.sql similarity index 86% rename from src/utils/calc_stats.sql rename to src/query/calc_stats.sql index 7ca0a2a..c8173e1 100644 --- a/src/utils/calc_stats.sql +++ b/src/query/calc_stats.sql @@ -27,16 +27,6 @@ FROM perceptual_model GROUP BY num_spatial_zones ORDER BY num_spatial_zones; -/* - num_spatial_zones | count --------------------+------- - 1 | 51 - 2 | 7 - 4 | 2 - 6 | 2 - 7 | 1 -*/ - -- count by temporal zones SELECT id, num_temporal_zones FROM perceptual_model @@ -47,16 +37,6 @@ FROM perceptual_model GROUP BY num_temporal_zones ORDER BY num_temporal_zones; -/* - num_temporal_zones | count ---------------------+------- - 1 | 35 - 2 | 11 - 3 | 8 - 4 | 8 - 5 | 1 -*/ - -- count by flux WITH flux_count AS @@ -100,21 +80,6 @@ FROM flux_count GROUP BY num_flux_per_model ORDER BY num_flux_per_model; -/* - num_flux_per_model | num_models ---------------------+------------ - 1 | 1 - 2 | 5 - 3 | 9 - 4 | 17 - 5 | 10 - 6 | 7 - 7 | 7 - 8 | 4 - 9 | 1 -*/ - - -- count by stores WITH store_count AS (SELECT perceptual_model.id, COUNT(perceptual_model.id) AS num_store_per_model @@ -154,19 +119,6 @@ SELECT num_store_per_model, COUNT(num_store_per_model) AS num_models FROM store_count GROUP BY num_store_per_model ORDER BY num_store_per_model; -/* - num_store_per_model | num_models ----------------------+------------ - 1 | 2 - 2 | 2 - 3 | 11 - 4 | 20 - 5 | 8 - 6 | 6 - 7 | 4 - 8 | 2 - 9 | 3 -*/ -- 2. Count by process @@ -338,11 +290,3 @@ FROM process_table COUNT(identifier) FROM process_table GROUP BY CHAR_LENGTH(identifier) - CHAR_LENGTH(REPLACE(identifier, '.', '')); - -/* - process_level | count ----------------+------- - 3 | 177 - 2 | 292 - 1 | 85 - */ \ No newline at end of file diff --git a/src/utils/debug_built_database.sql b/src/query/debug_built_database.sql similarity index 100% rename from src/utils/debug_built_database.sql rename to src/query/debug_built_database.sql