update readme & clean

RY4GIT · Dec 17, 2024 · b060d6f · b060d6f
1 parent b3fb43e
commit b060d6f
Show file tree

Hide file tree

Showing 16 changed files with 388 additions and 45 deletions.
diff --git a/README.md b/README.md
@@ -1,38 +1,72 @@
 # smap-drydown
 This repository contains code for analyzing soil moisture drydowns from [the Soil Moisture Active Passive (SMAP)](https://smap.jpl.nasa.gov/data/) as detailed in the corresponding manuscript: 
 
-    Araki, R., Morgan, B.E., McMillan, H.K., Caylor, K.K. Nonlinear Soil Moisture Loss Function Reveals Vegetation Responses to Water Availability. Geophysical Research Letters (in review)
+    Araki, R., Morgan, B.E., McMillan, H.K., Caylor, K.K. Nonlinear Soil Moisture Loss Function Reveals Vegetation Responses to Water Availability. Submitted to Geophysical Research Letters (in review)
 
 ## Getting started
 1. Clone the repository
 ```bash
 $ git clone [email protected]:RY4GIT/smap-drydown.git
 ```
 
-2. Create a virtual environment (select appropriate yml file according to your OS). 
+2. Create a virtual environment (select an appropriate yml file according to your OS). 
 ```bash
 $ cd smap-drydown
 $ conda env create -f environment_linux.yml
 $ conda activate SMAP
 ```
 
-3. Download the SMAP and ancillary data from appropriate sources.
+3. Download the SMAP and ancillary data from appropriate sources using scripts in `data_mng`
 
 4. In the `analysis` directory, create `config.ini`, based on `config_example.ini`
 
 5. Run `analysis\__main__.py`
 
-## Contents
+6. Visualize the results using scripts in `notebooks`. The results file is large (~130 MB) and is therefore available upon request.
 
-### `data_mng` 
-Contains scripts to retrieve & curate input data.
+## Contents
 
 ### `analysis`
 Contains scripts to implement the drydown analysis and model fits. 
-This code has been further refactored in https://github.com/ecohydro/drydowns: checkout if you are interested in. 
+
+The functions for loss calculations and drydown models are contained in `DrydownModel.py`. 
+This code has been further refactored in https://github.com/ecohydro/drydowns; check it out if you are interested.
+
+
+### `data_mng` 
+Contains scripts to retrieve and curate input data. 
+
+All data are pre-curated in "datarods" format, which stores the data as a long time series at a single SMAP grid.
+
+- SMAP soil moisture data
+    - Download data using `retrieve_NSIDC_Data_SPL3SMP.ipynb`
+    - Preprocess data using `create_datarods_SPL3SMP.ipynb`
+- SMAP precipitation data
+    - Download data using `retrieve_NSIDC_Data_SPL4SMGP.ipynb`
+    - Preprocess data using `create_datarods_SPL4SMGP.py`
+- dPET (Singer et al., 2020) data
+    - Download daily data from [the website](http://doi.org/10.5523/bris.qb8ujazzda0s2aykkv0oq0ctp)
+    - Preprocess data using `create_datarods_PET.py`
+- SMAP ancillary data
+    - Download data from [the website](https://doi.org/10.5067/HB8BPJ13TDQJ)
+    - Preprocess data using `read_ancillary_landcover_data.ipynb`
+    - After obtaining precipitation and PET data, run `calc_aridityindex.py`
+- Rangeland data
+    - Download data using `retrieve_rangeland_data.sh`
+    - Preprocess data using `read_rangeland_data.py`
+- Other utilities
+    - `retrieve_NSIDC_Data_datacheck.ipynb`: check if all the data are downloaded from NSIDC
+    - `create_datarods_datacheck.ipynb`: check if all the data are preprocessed
+    - `identify_unusable_grid.ipynb`: identify grids located on open water
+
 
 ### `notebooks`
-Contains scripts that are used to test functions or visualise results
+Contains scripts used to test functions or visualize the models and results
+- `figs_stats_datapreprocess.py`: Preprocess result files to reduce execution time
+- `figs_method.py` & `figs_method_tau.py`: Visualize the loss functions and drydown models
+- `figs_stats.py` and `figs_stats_rev.py`: Visualize the results
+- `figs_drydown.py`: Plot observed and modeled drydown curves
+
 
 ## Contact
-Ryoko Araki, raraki8150 (at) sdsu.edu
+Ryoko Araki, raraki8159 (at) sdsu.edu
diff --git a/data_mng/read_MODISvegetation.ipynb → data_mng/archive/read_MODISvegetation.ipynb b/data_mng/read_MODISvegetation.ipynb → data_mng/archive/read_MODISvegetation.ipynb
diff --git a/data_mng/read_bassiouni_data.ipynb → data_mng/archive/read_bassiouni_data.ipynb b/data_mng/read_bassiouni_data.ipynb → data_mng/archive/read_bassiouni_data.ipynb
diff --git a/data_mng/retrieve_MODIS44B.sh → data_mng/archive/retrieve_MODIS44B.sh b/data_mng/retrieve_MODIS44B.sh → data_mng/archive/retrieve_MODIS44B.sh
diff --git a/...ieve_MODIS_LAI_from_Appeears_pt_sample.py → ...ieve_MODIS_LAI_from_Appeears_pt_sample.py b/...ieve_MODIS_LAI_from_Appeears_pt_sample.py → ...ieve_MODIS_LAI_from_Appeears_pt_sample.py
diff --git a/...e_MODIS_NDVI_from_Appears_pt_sample.ipynb → ...e_MODIS_NDVI_from_Appears_pt_sample.ipynb b/...e_MODIS_NDVI_from_Appears_pt_sample.ipynb → ...e_MODIS_NDVI_from_Appears_pt_sample.ipynb
diff --git a/data_mng/retrieve_NSIDC_Data_SPL3SMP_E.ipynb → ...chive/retrieve_NSIDC_Data_SPL3SMP_E.ipynb b/data_mng/retrieve_NSIDC_Data_SPL3SMP_E.ipynb → ...chive/retrieve_NSIDC_Data_SPL3SMP_E.ipynb
diff --git a/data_mng/retrieve_appeears_smap_data.py → ...ng/archive/retrieve_appeears_smap_data.py b/data_mng/retrieve_appeears_smap_data.py → ...ng/archive/retrieve_appeears_smap_data.py
diff --git a/notebooks/check_ancillary_info.ipynb → notebooks/archive/check_ancillary_info.ipynb b/notebooks/check_ancillary_info.ipynb → notebooks/archive/check_ancillary_info.ipynb
@@ -1960,7 +1960,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "SMAP",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -1974,9 +1974,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.2"
+   "version": "3.12.3"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }
diff --git a/notebooks/figs_stats_agu24.py → notebooks/archive/figs_stats_agu24.py b/notebooks/figs_stats_agu24.py → notebooks/archive/figs_stats_agu24.py
diff --git a/notebooks/test_covmat.py → notebooks/archive/test_covmat.py b/notebooks/test_covmat.py → notebooks/archive/test_covmat.py
diff --git a/...test_event_separation_w_rainfall_v2.ipynb → ...test_event_separation_w_rainfall_v2.ipynb b/...test_event_separation_w_rainfall_v2.ipynb → ...test_event_separation_w_rainfall_v2.ipynb
diff --git a/notebooks/toy_drydown_models.ipynb → notebooks/archive/toy_drydown_models.ipynb b/notebooks/toy_drydown_models.ipynb → notebooks/archive/toy_drydown_models.ipynb
diff --git a/notebooks/figs_drydown.py b/notebooks/figs_drydown.py
@@ -22,8 +22,10 @@
 # %% Plot config
 
 ############ CHANGE HERE FOR CHECKING DIFFERENT RESULTS ###################
-dir_name = f"raraki_2024-05-13_global_piecewise"  # f"raraki_2024-04-26"
+dir_name = f"raraki_2024-12-03_revision"  # f"raraki_2024-04-26"
 ###########################################################################
+# f"raraki_2024-05-13_global_piecewise" was used for the 1st version of the manuscript
+# f"raraki_2024-12-03_revision" was used for the revised version of the manuscript
 
 ################ CHANGE HERE FOR PLOT VISUAL CONFIG #########################
 
@@ -95,7 +97,7 @@ def get_dataframe(varname, event):
         EASE_row_index=event.EASE_row_index,
         EASE_column_index=event.EASE_column_index,
     )
-    _df = pd.read_csv(os.path.join(data_dir, datarod_dir, varname, fn))
+    _df = pd.read_csv(os.path.join(data_dir, datarods_dir, varname, fn))
 
     # Set time index and crop
     df = set_time_index(_df, index_name="time")
@@ -598,21 +600,55 @@ def plot_drydown(
 print(f"Next to try (in df): {df_filt.sample(n=1).index}")
 print(f"Next to try (not in df): {not_in_filt_indices.to_series().sample(n=1).index}")
 # %%
-# # check_1ts_range(df.loc[event_id], verbose=True)
-# # %%
-# plt.scatter(df["event_length"], df["q_q"])
-# plt.scatter(df_filt["event_length"], df_filt["q_q"])
-
-# # %%
-# plt.scatter(df["large_q_criteria"], df["q_q"])
-# plt.scatter(df_filt["large_q_criteria"], df_filt["q_q"])
-# # %%
-# # %%
-# df_filt["q_r_squared"].hist()
-# plt.xlim([0, 1])
-# # %%
-# plt.scatter(df_filt["q_q"], df_filt["q_ETmax"])
-# # %%
-# ax = df["q_q"].hist(vmin=0, vmax=4)
-# ax.set_xlim([0, 3])
+
+# %%
+##########################################################
+# Select plot for revision
+
+# # CONUS
+# lat_min, lat_max = 24.396308, 49.384358
+# lon_min, lon_max = -125.000000, -66.934570
+# Define bounding box coordinates
+lon_min, lon_max = 67.507970, 89.653185
+lat_min, lat_max = 9.279354, 31.365056
+
+df_filt = df[
+    (df["q_r_squared"] < df["tauexp_r_squared"])
+    & (df["q_r_squared"] < 0.95)
+    & (df["q_r_squared"] > 0.8)
+    & (df["sm_range"] > 0.20)
+    & (df["large_q_criteria"] < 0.8)
+    & (df["first3_avail2"])
+    & (df["q_q"] > 1.0e-04)
+    & (df["event_length"] >= 7)
+    & (df["AI"] < 0.5)
+    # & (df["latitude"] >= lat_min)
+    # & (df["latitude"] <= lat_max)
+    # & (df["longitude"] >= lon_min)
+    # & (df["longitude"] <= lon_max)
+    # & (df["q_q"] > 1.7)
+    # & (df["q_q"] < 2.0)
+    & (df["sand_fraction"] > 0.7)
+]
+# print(df_filt)
+print(f"Try (in df): {df_filt.sample(n=5).index}")
+
+# %%
+df_filt.columns
+# %%|
+################################################
+event_id = 304485
+################################################
+
+# Arid example: 462921, 127783, 602222, 218603: 816131, 304485, 320094
+# Cropland example: 506447, 484442,
+
+save = True
+plot_drydown(
+    df=df, event_id=event_id, plot_precip=False, save=save, days_after_to_plot=13
+)
+
+# print(df.loc[event_id])
+print(f"Next to try (in df): {df_filt.sample(n=1).index}")
+print(f"Next to try (not in df): {not_in_filt_indices.to_series().sample(n=1).index}")
 # %%
diff --git a/notebooks/figs_stats_datapreprocess.py b/notebooks/figs_stats_datapreprocess.py
@@ -37,7 +37,7 @@
 ############################|###############################################
 # f"raraki_2024-05-13_global_piecewise" was used for the 1st version of the manuscript
 
-# Ryoko Araki, Bryn Morgan, Hilary K McMillan, Kelly K Caylor. 
+# Ryoko Araki, Bryn Morgan, Hilary K McMillan, Kelly K Caylor.
 # Nonlinear Soil Moisture Loss Function Reveals Vegetation Responses to Water Availability. ESS Open Archive . August 01, 2024.
 # DOI: 10.22541/essoar.172251989.99347091/v1
 
@@ -137,6 +137,7 @@
 df = df.assign(diff_bic_q_tauexp=df["q_bic"] - df["tauexp_bic"])
 df = df.assign(diff_bic_q_exp=df["q_bic"] - df["exp_bic"])
 
+
 def df_availability(row):
     # Check df point availability in the first 3 time steps of observation
     # Define a helper function to convert string to list