Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: remove save_rank_output, save_soilgrids_output, load_model_output, save_model_output from global soil #130

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

fix: remove save_rank_output, save_soilgrids_output, load_model_output, save_model_output from global soil #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
024ab67
fix: remove save_rank_output, save_soilgrids_output, load_model_outpu…
paulschreiber Jul 2, 2024
6132e51
fix: fix linter errors
paulschreiber Jul 2, 2024
6463e3c
fix: remove db calls from services
paulschreiber Jul 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 0 additions & 109 deletions soil_id/db.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,115 +31,6 @@ def get_datastore_connection():
sys.exit(str(err))


# us, global
def save_model_output(
plot_id, model_version, result_blob, soilIDRank_output_pd, mucompdata_cond_prob
):
"""
Save the output of the model to the 'landpks_soil_model' table.
"""
try:
conn = get_datastore_connection()
cur = conn.cursor()

sql = """
INSERT INTO landpks_soil_model
(plot_id, model_version, result_blob, soilIDRank_output_pd, mucompdata_cond_prob)
VALUES (%s, %s, %s, %s, %s)
"""
cur.execute(
sql,
(
plot_id,
model_version,
result_blob,
soilIDRank_output_pd,
mucompdata_cond_prob,
),
)
conn.commit()

except Exception as err:
logging.error(err)
conn.rollback()
return None
finally:
conn.close()


# us, global
def save_rank_output(record_id, model_version, rank_blob):
"""
Update the rank of the soil model in the 'landpks_soil_model' table.
"""
try:
conn = get_datastore_connection()
cur = conn.cursor()

sql = """UPDATE landpks_soil_model
SET soilrank = %s
WHERE ID = %s AND model_version = %s"""
cur.execute(sql, (rank_blob, record_id, model_version))
conn.commit()

except Exception as err:
logging.error(err)
conn.rollback()
return None
finally:
conn.close()


# us, global
def load_model_output(plot_id):
"""
Load model output based on plot ID and model version.
"""
try:
conn = get_datastore_connection()
cur = conn.cursor()
model_version = 2
sql = """SELECT ID, result_blob, soilIDRank_output_pd, mucompdata_cond_prob
FROM landpks_soil_model
WHERE plot_id = %s AND model_version = %s
ORDER BY ID DESC LIMIT 1"""
cur.execute(sql, plot_id, model_version)
results = cur.fetchall()
for row in results:
model_run = [row[0], row[1], row[2], row[3]]
return model_run
except Exception as err:
logging.error(err)
return None
finally:
conn.close()


# global only
def save_soilgrids_output(plot_id, model_version, soilgrids_blob):
"""
Save the output of the soil grids to the 'landpks_soilgrids_model' table.
"""
try:
conn = get_datastore_connection()
cur = conn.cursor()

sql = """
INSERT INTO landpks_soilgrids_model
(plot_id, model_version, soilgrids_blob)
VALUES (%s, %s, %s)
"""
cur.execute(sql, (plot_id, model_version, soilgrids_blob))
conn.commit()

except Exception as err:
logging.error(err)
conn.rollback()
return None
finally:
conn.close()


# global (via extract_WISE_data)
def get_WISE30sec_data(MUGLB_NEW_Select):
"""
Expand Down
88 changes: 9 additions & 79 deletions soil_id/global_soil.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
# Standard libraries
import collections
import csv
import io
import json
import re

# local libraries
Expand All @@ -11,14 +9,7 @@
# Third-party libraries
import numpy as np
import pandas as pd
from db import (
get_WRB_descriptions,
getSG_descriptions,
load_model_output,
save_model_output,
save_rank_output,
save_soilgrids_output,
)
from db import get_WRB_descriptions, getSG_descriptions
from scipy.stats import norm
from services import get_soilgrids_classification_data, get_soilgrids_property_data
from utils import (
Expand Down Expand Up @@ -57,7 +48,7 @@
##################################################################################################
# getSoilLocationBasedGlobal #
##################################################################################################
def getSoilLocationBasedGlobal(lon, lat, plot_id):
def getSoilLocationBasedGlobal(lon, lat):
# Extract HWSD-WISE Data
# Note: Need to convert HWSD shp to gpkg file
wise_data = extract_WISE_data(
Expand Down Expand Up @@ -363,7 +354,6 @@ def aggregated_data_layer(data, column_name):
compName = mucompdata_cond_prob["compname_grp"].apply(lambda x: x.capitalize()).tolist()
score = mucompdata_cond_prob["distance_score_norm"].round(3).tolist()
rank_loc = mucompdata_cond_prob["Rank_Loc"].tolist()
model_version = 3

# Step 3: Construct ID list directly from the sorted and cleaned DataFrame
ID = []
Expand Down Expand Up @@ -447,34 +437,8 @@ def aggregated_data_layer(data, column_name):
]

# Save data
if plot_id is None:
soilIDRank_output_pd.to_csv(config.SOIL_ID_RANK_PATH, index=None, header=True)
mucompdata_cond_prob.to_csv(config.SOIL_ID_PROB_PATH, index=None, header=True)
else:
save_model_output(
plot_id,
model_version,
json.dumps(
{
"metadata": {
"location": "global",
"model": "v2",
"unit_measure": {
"distance": "m",
"depth": "cm",
"cec": "cmol(c)/kg",
"clay": "%",
"rock_fragments": "cm3/100cm3",
"sand": "%",
"ec": "ds/m",
},
},
"soilList": output_SoilList,
}
),
soilIDRank_output_pd.to_csv(index=None, header=True),
mucompdata_cond_prob.to_csv(index=None, header=True),
)
soilIDRank_output_pd.to_csv(config.SOIL_ID_RANK_PATH, index=None, header=True)
mucompdata_cond_prob.to_csv(config.SOIL_ID_PROB_PATH, index=None, header=True)

# Return the JSON output
return {
Expand All @@ -498,9 +462,7 @@ def aggregated_data_layer(data, column_name):
##############################################################################################
# rankPredictionGlobal #
##############################################################################################
def rankPredictionGlobal(
lon, lat, soilHorizon, horizonDepth, rfvDepth, lab_Color, bedrock, cracks, plot_id=None
):
def rankPredictionGlobal(lon, lat, soilHorizon, horizonDepth, rfvDepth, lab_Color, bedrock, cracks):
# ------------------------------------------------------------------------------------------------
# ------ Load in user data --------#

Expand Down Expand Up @@ -658,25 +620,9 @@ def rankPredictionGlobal(

# --------------------------------------------------------------------------------------------------------------------------------------
# Load in component data from soilIDList
# Initialize
record_id = None

# If no plot_id is provided, load data from file
if plot_id is None:
soilIDRank_output_pd = pd.read_csv(config.SOIL_ID_RANK_PATH)
mucompdata_pd = pd.read_csv(config.SOIL_ID_PROB_PATH)

# If plot_id is provided, load data from the database
else:
modelRun = load_model_output(plot_id)

# Check if modelRun data was successfully fetched
if modelRun:
record_id = modelRun[0]
soilIDRank_output_pd = pd.read_csv(io.StringIO(modelRun[2]))
mucompdata_pd = pd.read_csv(io.StringIO(modelRun[3]))
else:
return "Cannot find a plot with this ID"
soilIDRank_output_pd = pd.read_csv(config.SOIL_ID_RANK_PATH)
mucompdata_pd = pd.read_csv(config.SOIL_ID_PROB_PATH)

# Create soil depth DataFrame and subset component depths based on max user depth
# if no bedrock specified
Expand Down Expand Up @@ -1207,7 +1153,6 @@ def rankPredictionGlobal(
rank_list.append(rank_entry)

# Setting up the return data structure
model_version = 3
metadata = {
"location": "global",
"model": "v3",
Expand All @@ -1216,17 +1161,13 @@ def rankPredictionGlobal(

result = {"metadata": metadata, "soilRank": rank_list}

# Save data if record_id is provided
if record_id:
save_rank_output(record_id, model_version, json.dumps(result))

return result


##################################################################################################
# getSoilGridsGlobal #
##################################################################################################
def getSoilGridsGlobal(lon, lat, plot_id=None):
def getSoilGridsGlobal(lon, lat):

# Call soildgrids API
sg_out = get_soilgrids_property_data(lon, lat)
Expand Down Expand Up @@ -1262,8 +1203,6 @@ def getSoilGridsGlobal(lon, lat, plot_id=None):

# Check if all values in the specified columns are NaN
if sg_data_w[["sand", "clay", "cfvo"]].isnull().all().all():
if plot_id is not None:
save_soilgrids_output(plot_id, 1, json.dumps({"status": "unavailable"}))
return {"status": "unavailable"}
else:
# Apply the factor to the specific columns
Expand All @@ -1273,7 +1212,7 @@ def getSoilGridsGlobal(lon, lat, plot_id=None):
sg_data_w["silt"] = sg_data_w.apply(silt_calc, axis=1)
sg_data_w["texture"] = sg_data_w.apply(getTexture, axis=1)

sg_tax = get_soilgrids_classification_data(lon, lat, plot_id)
sg_tax = get_soilgrids_classification_data(lon, lat)

# If data was successfully fetched, process it
if sg_tax:
Expand Down Expand Up @@ -1323,9 +1262,6 @@ def getSoilGridsGlobal(lon, lat, plot_id=None):
texture_pd["silt"] = texture_pd.apply(silt_calc, axis=1)
texture_pd_lpks = texture_pd.apply(getTexture, axis=1).replace([None], "")

# SoilGrids API call version 1
model_version = 1

# Define keys and corresponding values for the 'components' dictionary
component_keys = [
"compname",
Expand Down Expand Up @@ -1392,11 +1328,5 @@ def getSoilGridsGlobal(lon, lat, plot_id=None):
},
}

# If a plot_id is provided, save the SoilGrids output
if plot_id is not None:
save_soilgrids_output(
plot_id, model_version, json.dumps({"metadata": metadata, "soilGrids": SoilGrids})
)

# Return the final result
return {"metadata": metadata, "soilGrids": SoilGrids}
9 changes: 0 additions & 9 deletions soil_id/services.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import json
import logging
import re

Expand All @@ -7,8 +6,6 @@
import requests
from pandas import json_normalize

from .db import save_soilgrids_output


def get_elev_data(lon, lat):
"""
Expand Down Expand Up @@ -157,9 +154,6 @@ def get_soilgrids_property_data(lon, lat, plot_id):
logging.error("Soilgrids properties: timed out")
except requests.RequestException as err:
logging.error(f"Soilgrids properties: error: {err}")
if plot_id is not None:
# Assuming the function `save_soilgrids_output` exists elsewhere in the code
save_soilgrids_output(plot_id, 1, json.dumps({"status": "unavailable"}))

return result if result is not None else {"status": "unavailable"}

Expand All @@ -182,9 +176,6 @@ def get_soilgrids_classification_data(lon, lat, plot_id):
logging.error("Soilgrids classification: timed out")
except requests.RequestException as err:
logging.error(f"Soilgrids classification: error: {err}")
if plot_id is not None:
# Assuming the function `save_soilgrids_output` exists elsewhere in the code
save_soilgrids_output(plot_id, 1, json.dumps({"status": "unavailable"}))

return result

Expand Down
Loading