From 831d66e664e2122d50b5fd633e19ea5c26e69a71 Mon Sep 17 00:00:00 2001 From: Manjila Singh Date: Mon, 12 Aug 2024 20:03:40 +0100 Subject: [PATCH 1/9] Change sidebar position order --- docs/products/forcingprocessor/forcingprocessor.md | 2 +- docs/products/nextgeninaboxDocker/index.md | 2 +- docs/products/ngiabclient/index.md | 2 +- docs/products/ngiabpreprocessor/index.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/products/forcingprocessor/forcingprocessor.md b/docs/products/forcingprocessor/forcingprocessor.md index 620f1d4..3a17091 100644 --- a/docs/products/forcingprocessor/forcingprocessor.md +++ b/docs/products/forcingprocessor/forcingprocessor.md @@ -1,5 +1,5 @@ --- -sidebar_position: 5 +sidebar_position: 6 title: "Forcing Processor" description: "NextGen Forcing Processor" tags: diff --git a/docs/products/nextgeninaboxDocker/index.md b/docs/products/nextgeninaboxDocker/index.md index 8b042ef..3e52da7 100644 --- a/docs/products/nextgeninaboxDocker/index.md +++ b/docs/products/nextgeninaboxDocker/index.md @@ -1,5 +1,5 @@ --- -sidebar_position: 1 +sidebar_position: 3 title: "NGIAB-CloudInfra" description: "NextGen In a Box (NGIAB)" tags: diff --git a/docs/products/ngiabclient/index.md b/docs/products/ngiabclient/index.md index 306cce7..4129ceb 100644 --- a/docs/products/ngiabclient/index.md +++ b/docs/products/ngiabclient/index.md @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 4 title: 'NGIAB Data Visualizer' description: 'Tethys Apps Portal' tags: diff --git a/docs/products/ngiabpreprocessor/index.md b/docs/products/ngiabpreprocessor/index.md index 25517a8..84d328b 100644 --- a/docs/products/ngiabpreprocessor/index.md +++ b/docs/products/ngiabpreprocessor/index.md @@ -1,5 +1,5 @@ --- -sidebar_position: 4 +sidebar_position: 2 title: "NGIAB-data-preprocessor" description: "NextGen Data Preprocessor" tags: From 38dee5a19effb977bb994c369dcdfead2f03497c Mon Sep 17 00:00:00 2001 From: Manjila Singh Date: Mon, 12 Aug 2024 20:04:06 +0100 Subject: [PATCH 2/9] Add NextGen hydrofabric link --- docs/education/hydrofabric/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/education/hydrofabric/index.md b/docs/education/hydrofabric/index.md index 0e0b818..05ce577 100644 --- a/docs/education/hydrofabric/index.md +++ b/docs/education/hydrofabric/index.md @@ -36,4 +36,5 @@ To get started with Hydrofabric, you can follow these steps: ## Hydrofabric Resources - [Hydrofabric Code Repository](https://github.com/NOAA-OWP/hydrofabric) - [Hydrofabric Data](https://www.lynker-spatial.com/) +- [NextGen Hydrofabric](https://mikejohnson51.github.io/hyAggregate/) From e5e4f722e522da6a871b22ce3520b8666c9f8ca4 Mon Sep 17 00:00:00 2001 From: Manjila Singh Date: Mon, 12 Aug 2024 20:04:27 +0100 Subject: [PATCH 3/9] Add bmi basics page --- docs/education/NextGenFramework/bmi.md | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/education/NextGenFramework/bmi.md diff --git a/docs/education/NextGenFramework/bmi.md b/docs/education/NextGenFramework/bmi.md new file mode 100644 index 0000000..1016d6e --- /dev/null +++ b/docs/education/NextGenFramework/bmi.md @@ -0,0 +1,29 @@ +--- +sidebar_position: 4 +title: "BMI Basics for NextGen" +description: "bmi" +tags: +- NWM +- BMI + +--- + +BMI is a middleware, a software that operates between other components (in this case, NextGen). It standardizes model querying, control, and coupling. + +From the [BMI wiki](https://bmi.readthedocs.io/en/stable/): +
+We believe that numerical models, and the sub-components that make up those models, +should offer....standardization. To this end, the Community Surface Dynamics Modeling +System (CSDMS) has developed the Basic Model Interface (BMI): a set of **standard +control and query functions** that, when added to a model code, make that model both +easier to learn and easier to couple with other software elements. +
+ +## Key Features of BMI +- Control model runtime using standardized functions +- Pass data in/out of models using standardized functions +- Provide model and variable information +- Work in multiple languages and models. + +## Resources +- Conference material: https://github.com/SnowHydrology/ciroh_workshop_2023/blob/main/presentation/bmi_basics.pdf From b7dc952b3d1e23f66a81316d9713f996f9cf632a Mon Sep 17 00:00:00 2001 From: Manjila Singh Date: Mon, 12 Aug 2024 20:22:04 +0100 Subject: [PATCH 4/9] Add best practices for GCP --- .../google-cloud/best-practices/index.md | 94 +++++++++++++++++++ .../{google cloud => google-cloud}/index.md | 3 +- docusaurus.config.js | 6 +- 3 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 docs/services/cloudservices/google-cloud/best-practices/index.md rename docs/services/cloudservices/{google cloud => google-cloud}/index.md (99%) diff --git a/docs/services/cloudservices/google-cloud/best-practices/index.md b/docs/services/cloudservices/google-cloud/best-practices/index.md new file mode 100644 index 0000000..7371c05 --- /dev/null +++ b/docs/services/cloudservices/google-cloud/best-practices/index.md @@ -0,0 +1,94 @@ +--- +sidebar_position: 2 +title: "GCP Best Practices" +description: "Best Practices for working with gcp storage buckets" +tags: + - Google + - Cloud + - CIROH + - GCP + - best practices + - storage buckets + +--- + +# Best Practices for Working with GCP Storage Buckets +Given your current access permissions, this guide will help you work effectively and safely with Google Cloud Storage buckets. + +#### Understanding Your Permissions +You have been granted the following permissions: +- Storage Object Creator +- Storage Object User +- Storage Object Viewer +These permissions allow you to create buckets, list existing buckets, create objects within buckets, and list/read objects. + +## Best Practices +### 1. Naming Conventions +- Use clear, descriptive names for buckets and objects +- Avoid using sensitive information in names +- Follow a consistent naming pattern (e.g., project-environment-purpose) + +### 2. Bucket Creation +- Before creating a new bucket, check if an appropriate one already exists +- Document the purpose of each bucket you create +- Set appropriate region/location based on data residency requirements +### 3. Object Management +- Organize objects using a logical folder structure +- Use object versioning for critical data +- Set appropriate object lifecycle rules to manage storage costs +### 4. Data Protection +- Enable object versioning for important data +- Set up retention policies for regulatory compliance if needed +- Use client-side encryption for sensitive data before uploading +### 5. Cost Management +- Monitor storage usage regularly +- Set up budget alerts to avoid unexpected costs +- Use appropriate storage classes based on access patterns +### 6. Performance +- Choose the appropriate location for your buckets to minimize latency +- Use composite objects for large files (>5GB) +- Implement retry logic in your applications for better resilience +### 7. Logging and Monitoring +- Enable Cloud Audit Logs for bucket and object-level activities +- Set up monitoring and alerting for unusual activities or errors +### 8. Cleanup and Maintenance +- Regularly review and remove unnecessary objects and buckets +- Archive or delete old versions of objects if versioning is enabled +### 9. Security +- Use signed URLs for temporary access to objects +- Implement strong authentication mechanisms in your applications +- Avoid storing sensitive data in object metadata +### 10. Accessing Buckets in Code +- Use Application Default Credentials (ADC) for authentication in your code +- Follow these steps to set up ADC: a. Ensure you're logged into your Google account in the browser b. Install the Google Cloud SDK on your local machine c. +- Run gcloud auth application-default login in the terminal d. This authenticates you and stores the credentials locally +- In your Python code, use the following to access buckets: +``` python +from google.auth import default +from google.cloud import storage +credentials, project = default() +fs = gcsfs.GCSFileSystem(token=credentials) +``` +### 11. Code Development Best Practices +- Use version control (e.g., Git) for all your code +- Implement proper error handling and logging in your scripts +- Follow Python (or your chosen language) best practices and style guides +- Use environment variables or configuration files for bucket names and other changeable parameters +- Implement unit tests for your bucket interaction code +- Use type hinting in Python for better code readability and catch potential errors early +### 12. Security in Code +- Never hardcode credentials in your scripts +- Use secret management solutions for storing sensitive information +- Implement proper exception handling to avoid exposing sensitive information in error messages +- Regularly update your dependencies to patch known vulnerabilities +- Use HTTPS for all network communications + + +Remember: Your actions can affect the entire organization's storage infrastructure. When in doubt, consult with your team lead or GCP administrator. + + + + + +mkdir -p NextGen/ngen-data +cd NextGen/ngen-data \ No newline at end of file diff --git a/docs/services/cloudservices/google cloud/index.md b/docs/services/cloudservices/google-cloud/index.md similarity index 99% rename from docs/services/cloudservices/google cloud/index.md rename to docs/services/cloudservices/google-cloud/index.md index 135495d..e1a9545 100644 --- a/docs/services/cloudservices/google cloud/index.md +++ b/docs/services/cloudservices/google-cloud/index.md @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 1 title: "CIROH Google Account" description: "Google Research Cloud" tags: @@ -12,7 +12,6 @@ tags: # CIROH Google Account - CIROH Cloud leverages the power of Google Cloud to empower researchers and unlock groundbreaking advancements in hydrology. CIROH provides access to enterprise-level Google cloud platform to researchers.
diff --git a/docusaurus.config.js b/docusaurus.config.js index 4617a08..ed8e093 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -172,7 +172,11 @@ plugins: [[ require.resolve('docusaurus-lunr-search'), { ], }, ], - copyright: `Copyright © ${new Date().getFullYear()} CIROH - The University of Alabama`, + copyright: `

+ his project received funding under award NA22NWS4320003 from NOAA Cooperative Institute Program. The statements, findings, conclusions, and recommendations are those of the author(s) and do not necessarily reflect the views of NOAA. +


+ Copyright © ${new Date().getFullYear()} CIROH - The University of Alabama + `, }, prism: { theme: prismThemes.github, From 187cac8917e60e1f6b782c73dcbf721e5a726436 Mon Sep 17 00:00:00 2001 From: Manjila Singh Date: Tue, 13 Aug 2024 14:13:30 +0100 Subject: [PATCH 5/9] Revert "Change sidebar position order" This reverts commit 831d66e664e2122d50b5fd633e19ea5c26e69a71. --- docs/products/forcingprocessor/forcingprocessor.md | 2 +- docs/products/nextgeninaboxDocker/index.md | 2 +- docs/products/ngiabclient/index.md | 2 +- docs/products/ngiabpreprocessor/index.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/products/forcingprocessor/forcingprocessor.md b/docs/products/forcingprocessor/forcingprocessor.md index 3a17091..620f1d4 100644 --- a/docs/products/forcingprocessor/forcingprocessor.md +++ b/docs/products/forcingprocessor/forcingprocessor.md @@ -1,5 +1,5 @@ --- -sidebar_position: 6 +sidebar_position: 5 title: "Forcing Processor" description: "NextGen Forcing Processor" tags: diff --git a/docs/products/nextgeninaboxDocker/index.md b/docs/products/nextgeninaboxDocker/index.md index 3e52da7..8b042ef 100644 --- a/docs/products/nextgeninaboxDocker/index.md +++ b/docs/products/nextgeninaboxDocker/index.md @@ -1,5 +1,5 @@ --- -sidebar_position: 3 +sidebar_position: 1 title: "NGIAB-CloudInfra" description: "NextGen In a Box (NGIAB)" tags: diff --git a/docs/products/ngiabclient/index.md b/docs/products/ngiabclient/index.md index 4129ceb..306cce7 100644 --- a/docs/products/ngiabclient/index.md +++ b/docs/products/ngiabclient/index.md @@ -1,5 +1,5 @@ --- -sidebar_position: 4 +sidebar_position: 2 title: 'NGIAB Data Visualizer' description: 'Tethys Apps Portal' tags: diff --git a/docs/products/ngiabpreprocessor/index.md b/docs/products/ngiabpreprocessor/index.md index 84d328b..25517a8 100644 --- a/docs/products/ngiabpreprocessor/index.md +++ b/docs/products/ngiabpreprocessor/index.md @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 4 title: "NGIAB-data-preprocessor" description: "NextGen Data Preprocessor" tags: From d7ae53c3bd3e1a3be4050a53fc66123ac2ea52af Mon Sep 17 00:00:00 2001 From: Manjila Singh Date: Wed, 14 Aug 2024 14:56:16 +0100 Subject: [PATCH 6/9] Revert "Add best practices for GCP" This reverts commit b7dc952b3d1e23f66a81316d9713f996f9cf632a. --- .../{google-cloud => google cloud}/index.md | 3 +- .../google-cloud/best-practices/index.md | 94 ------------------- docusaurus.config.js | 6 +- 3 files changed, 3 insertions(+), 100 deletions(-) rename docs/services/cloudservices/{google-cloud => google cloud}/index.md (99%) delete mode 100644 docs/services/cloudservices/google-cloud/best-practices/index.md diff --git a/docs/services/cloudservices/google-cloud/index.md b/docs/services/cloudservices/google cloud/index.md similarity index 99% rename from docs/services/cloudservices/google-cloud/index.md rename to docs/services/cloudservices/google cloud/index.md index e1a9545..135495d 100644 --- a/docs/services/cloudservices/google-cloud/index.md +++ b/docs/services/cloudservices/google cloud/index.md @@ -1,5 +1,5 @@ --- -sidebar_position: 1 +sidebar_position: 2 title: "CIROH Google Account" description: "Google Research Cloud" tags: @@ -12,6 +12,7 @@ tags: # CIROH Google Account + CIROH Cloud leverages the power of Google Cloud to empower researchers and unlock groundbreaking advancements in hydrology. CIROH provides access to enterprise-level Google cloud platform to researchers.
diff --git a/docs/services/cloudservices/google-cloud/best-practices/index.md b/docs/services/cloudservices/google-cloud/best-practices/index.md deleted file mode 100644 index 7371c05..0000000 --- a/docs/services/cloudservices/google-cloud/best-practices/index.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -sidebar_position: 2 -title: "GCP Best Practices" -description: "Best Practices for working with gcp storage buckets" -tags: - - Google - - Cloud - - CIROH - - GCP - - best practices - - storage buckets - ---- - -# Best Practices for Working with GCP Storage Buckets -Given your current access permissions, this guide will help you work effectively and safely with Google Cloud Storage buckets. - -#### Understanding Your Permissions -You have been granted the following permissions: -- Storage Object Creator -- Storage Object User -- Storage Object Viewer -These permissions allow you to create buckets, list existing buckets, create objects within buckets, and list/read objects. - -## Best Practices -### 1. Naming Conventions -- Use clear, descriptive names for buckets and objects -- Avoid using sensitive information in names -- Follow a consistent naming pattern (e.g., project-environment-purpose) - -### 2. Bucket Creation -- Before creating a new bucket, check if an appropriate one already exists -- Document the purpose of each bucket you create -- Set appropriate region/location based on data residency requirements -### 3. Object Management -- Organize objects using a logical folder structure -- Use object versioning for critical data -- Set appropriate object lifecycle rules to manage storage costs -### 4. Data Protection -- Enable object versioning for important data -- Set up retention policies for regulatory compliance if needed -- Use client-side encryption for sensitive data before uploading -### 5. Cost Management -- Monitor storage usage regularly -- Set up budget alerts to avoid unexpected costs -- Use appropriate storage classes based on access patterns -### 6. Performance -- Choose the appropriate location for your buckets to minimize latency -- Use composite objects for large files (>5GB) -- Implement retry logic in your applications for better resilience -### 7. Logging and Monitoring -- Enable Cloud Audit Logs for bucket and object-level activities -- Set up monitoring and alerting for unusual activities or errors -### 8. Cleanup and Maintenance -- Regularly review and remove unnecessary objects and buckets -- Archive or delete old versions of objects if versioning is enabled -### 9. Security -- Use signed URLs for temporary access to objects -- Implement strong authentication mechanisms in your applications -- Avoid storing sensitive data in object metadata -### 10. Accessing Buckets in Code -- Use Application Default Credentials (ADC) for authentication in your code -- Follow these steps to set up ADC: a. Ensure you're logged into your Google account in the browser b. Install the Google Cloud SDK on your local machine c. -- Run gcloud auth application-default login in the terminal d. This authenticates you and stores the credentials locally -- In your Python code, use the following to access buckets: -``` python -from google.auth import default -from google.cloud import storage -credentials, project = default() -fs = gcsfs.GCSFileSystem(token=credentials) -``` -### 11. Code Development Best Practices -- Use version control (e.g., Git) for all your code -- Implement proper error handling and logging in your scripts -- Follow Python (or your chosen language) best practices and style guides -- Use environment variables or configuration files for bucket names and other changeable parameters -- Implement unit tests for your bucket interaction code -- Use type hinting in Python for better code readability and catch potential errors early -### 12. Security in Code -- Never hardcode credentials in your scripts -- Use secret management solutions for storing sensitive information -- Implement proper exception handling to avoid exposing sensitive information in error messages -- Regularly update your dependencies to patch known vulnerabilities -- Use HTTPS for all network communications - - -Remember: Your actions can affect the entire organization's storage infrastructure. When in doubt, consult with your team lead or GCP administrator. - - - - - -mkdir -p NextGen/ngen-data -cd NextGen/ngen-data \ No newline at end of file diff --git a/docusaurus.config.js b/docusaurus.config.js index ed8e093..4617a08 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -172,11 +172,7 @@ plugins: [[ require.resolve('docusaurus-lunr-search'), { ], }, ], - copyright: `

- his project received funding under award NA22NWS4320003 from NOAA Cooperative Institute Program. The statements, findings, conclusions, and recommendations are those of the author(s) and do not necessarily reflect the views of NOAA. -


- Copyright © ${new Date().getFullYear()} CIROH - The University of Alabama - `, + copyright: `Copyright © ${new Date().getFullYear()} CIROH - The University of Alabama`, }, prism: { theme: prismThemes.github, From c49301bb4721caa97301dcdefa15c945416fb40f Mon Sep 17 00:00:00 2001 From: Manjila Singh Date: Thu, 12 Sep 2024 21:25:35 +0100 Subject: [PATCH 7/9] Added plugin-google-gtag --- docusaurus.config.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docusaurus.config.js b/docusaurus.config.js index d172112..4ec38fd 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -28,6 +28,10 @@ const config = { "classic", /** @type {import('@docusaurus/preset-classic').Options} */ ({ + gtag: { + trackingID: 'G-TQW7CE7E2P', + anonymizeIP: true, + }, blog: { blogTitle: "DocuHub blog!", blogDescription: "A DocuHub powered blog!", From a788d781a8f14a7bde4137835a9425a0b31b8e74 Mon Sep 17 00:00:00 2001 From: Manjila Singh Date: Thu, 12 Sep 2024 21:28:10 +0100 Subject: [PATCH 8/9] Remove bmi.md --- docs/education/NextGenFramework/bmi.md | 29 -------------------------- 1 file changed, 29 deletions(-) delete mode 100644 docs/education/NextGenFramework/bmi.md diff --git a/docs/education/NextGenFramework/bmi.md b/docs/education/NextGenFramework/bmi.md deleted file mode 100644 index 1016d6e..0000000 --- a/docs/education/NextGenFramework/bmi.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -sidebar_position: 4 -title: "BMI Basics for NextGen" -description: "bmi" -tags: -- NWM -- BMI - ---- - -BMI is a middleware, a software that operates between other components (in this case, NextGen). It standardizes model querying, control, and coupling. - -From the [BMI wiki](https://bmi.readthedocs.io/en/stable/): -
-We believe that numerical models, and the sub-components that make up those models, -should offer....standardization. To this end, the Community Surface Dynamics Modeling -System (CSDMS) has developed the Basic Model Interface (BMI): a set of **standard -control and query functions** that, when added to a model code, make that model both -easier to learn and easier to couple with other software elements. -
- -## Key Features of BMI -- Control model runtime using standardized functions -- Pass data in/out of models using standardized functions -- Provide model and variable information -- Work in multiple languages and models. - -## Resources -- Conference material: https://github.com/SnowHydrology/ciroh_workshop_2023/blob/main/presentation/bmi_basics.pdf From 488411e066cac2c00748d7ae38c0430f64e33257 Mon Sep 17 00:00:00 2001 From: Manjila Singh Date: Thu, 12 Sep 2024 21:29:45 +0100 Subject: [PATCH 9/9] Update index.md --- docs/education/hydrofabric/index.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/education/hydrofabric/index.md b/docs/education/hydrofabric/index.md index 05ce577..0e0b818 100644 --- a/docs/education/hydrofabric/index.md +++ b/docs/education/hydrofabric/index.md @@ -36,5 +36,4 @@ To get started with Hydrofabric, you can follow these steps: ## Hydrofabric Resources - [Hydrofabric Code Repository](https://github.com/NOAA-OWP/hydrofabric) - [Hydrofabric Data](https://www.lynker-spatial.com/) -- [NextGen Hydrofabric](https://mikejohnson51.github.io/hyAggregate/)