From fa003e1aa014de019228d18d764fccdd77e463e2 Mon Sep 17 00:00:00 2001 From: ewiggansLI <117284551+ewiggansLI@users.noreply.github.com> Date: Tue, 26 Sep 2023 13:34:01 -0400 Subject: [PATCH] Add Wyoming gages (#10) * Create WyomingStreamGages_toGeoconnex.py * Add pygeoapi configuration for wyoming features * Squashed commit of the following: commit c7019d6d26cbe2c30eab7843c16139f026ab8a44 Author: Kyle Onda Date: Tue Aug 29 22:31:00 2023 -0400 Update features.config.yml commit 4c98ed3e0f0b32800b09ad89080a8736d081a86f Author: Kyle Onda Date: Tue Aug 29 22:16:37 2023 -0400 Update features.config.yml commit 357d93c3d2002797100441c66fd1aac960f21f0d Author: Kyle Onda Date: Tue Aug 29 21:57:15 2023 -0400 Update features.config.yml commit de9e170e78e63e1437f320822fd100891108c6f2 Author: Kyle Onda Date: Tue Aug 29 21:47:46 2023 -0400 Update features.config.yml commit ceaee48b732cd00b486a517d033bee28f3613c62 Author: Kyle Onda Date: Tue Aug 29 21:39:08 2023 -0400 Update features.config.yml commit 9bfe96b91cb1879b9f906d1302ffeabc88e02ff9 Author: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Thu Jul 6 11:05:59 2023 -0400 Add GitHub Actions for CI (#9) * Create deploy.yml * Remove dev test from CI commit 1f1f35aa279af157c0ce62b395c16ad3212abae9 Author: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Wed Jun 28 14:31:31 2023 +0200 Update pygeoapi configuration (#8) - Create valid bbox - Update data source url --------- Co-authored-by: Benjamin Webb --- .../wy/WyomingStreamGages_toGeoconnex.py | 246 ++++++++++++++++++ pygeoapi/features.config.yml | 22 ++ 2 files changed, 268 insertions(+) create mode 100644 collections/wy/WyomingStreamGages_toGeoconnex.py diff --git a/collections/wy/WyomingStreamGages_toGeoconnex.py b/collections/wy/WyomingStreamGages_toGeoconnex.py new file mode 100644 index 0000000..9f40d86 --- /dev/null +++ b/collections/wy/WyomingStreamGages_toGeoconnex.py @@ -0,0 +1,246 @@ +import arcpy + +# Set the workspace environment +arcpy.env.workspace = r"C:\Users\ewiggans\Desktop\GeoconnexNPDES\GeoconnexNPDES\GeoconnexNPDES.gdb" + +# Define the input layer +WY_gage = "WyomingStreamGage" + +# Define a list of field names and their types +field_list = [ + ("uri", "TEXT"), + ("name", "TEXT"), + ("id", "TEXT"), + ("provider_name", "TEXT"), + ("provider_id", "TEXT"), + ("provider_url", "TEXT"), + ("provider_code", "TEXT"), + ("mainstem_uri", "TEXT"), + ("comid", "TEXT") +] + +# Add the new fields using a loop +for field_name, field_type in field_list: + arcpy.AddField_management(WY_gage, field_name, field_type) + +# Create an update cursor to delete rows with NULL 'latitude' or 'longitude' +with arcpy.da.UpdateCursor(WY_gage, ['latitude', 'longitude']) as cursor: + for row in cursor: + latitude, longitude = row + if latitude is None or longitude is None: + cursor.deleteRow() + +# Confirm the field addition +field_names = [field.name for field in arcpy.ListFields(WY_gage)] +print("Fields in the layer after addition:", field_names) + + + + + +# Confirm unique ID in identifier field +# Field to check for uniqueness +field_name = "Identifier" + +# Use a set to keep track of unique values encountered in the field +unique_values = set() + +# Initialize a counter to keep track of the total number of rows +total_rows = 0 + +# Initialize a flag to check for spaces +has_spaces = False + +# Start a search cursor to iterate through the rows and check for unique values and spaces +with arcpy.da.UpdateCursor(WY_gage, field_name) as cursor: + for row in cursor: + total_rows += 1 + value = row[0] + if value in unique_values: + print(f"Non-unique value found: {value}") + else: + unique_values.add(value) + + if " " in value: + has_spaces = True + # Replace spaces with "-" + row[0] = value.replace(" ", "-") + cursor.updateRow(row) + +# Check if the number of unique values is the same as the total number of rows +if len(unique_values) == total_rows: + print("All values in the 'Identifier' field are unique.") +else: + print("There are duplicate values in the 'Identifier' field.") + +# Check if there are any spaces in the 'Identifier' field +if has_spaces: + print("There were spaces in the 'Identifier' field. They have been replaced with '-'.") +else: + print("No spaces found in the 'Identifier' field.") + + + +#set Identifier field to id and name +#fields to update +id_field = "id" +name_field = "name" +field_name = "Identifier" + +# Calculate the 'id' field and 'name' field using the 'Identifier' field +expression = "!" + field_name + "!" +codeblock = "" + +# Use the CalculateField tool to perform the field calculation +arcpy.CalculateField_management(WY_gage, id_field, expression, "PYTHON3", codeblock) +arcpy.CalculateField_management(WY_gage, name_field, expression, "PYTHON3", codeblock) + +#print("Updated 'id' and 'name' fields with values from the 'Identifier' field.") + +arcpy.CalculateField_management(WY_gage, "provider_name", "'Wyoming State Engineers Office'") +arcpy.CalculateField_management(WY_gage, "provider_url", "'https://seo.wyo.gov/'") +arcpy.CalculateField_management(WY_gage, "provider_id", "!" + field_name +"!") +arcpy.CalculateField_management(WY_gage, "provider_code", "'wyseo'") + + +#Calculate URI +codeblock = """ +def url_join(*parts: list) -> str: + return '/'.join([str(p).strip().strip('/') for p in parts]) +""" +## pattern for uri is: geoconnex.us/iow/wyseo/gages/id + +expression = """url_join("https://geoconnex.us/wyseo/gages", !id!)""" +arcpy.management.CalculateField(WY_gage, "uri", expression, "PYTHON3", codeblock) +print("uri calculated") + + + +##Read in the download table + +import pandas as pd +pd.set_option('display.width', 1000) + +# Path to the CSV file +csv_file_path = r"C:\Users\ewiggans\Desktop\GeoConnexMap\WyomingGage\List-20230801130904_WYgages.csv" + + +# Read the CSV file into a pandas DataFrame +df = pd.read_csv(csv_file_path) + +# Display the first few rows of the data frame to verify the data +print(df.head()) + + +#View column headers +df.columns + +# Perform the field calculations on the pandas DataFrame +df["location_id"] = df["Data_Set_Id"].str.replace(r".*@", "", regex=True) +df["about_url"] = "https://geoconnex.us/wyseo/gages/" + df["location_id"] +df["data_set_type"] = df["Data_Set_Id"].str.replace(r"@.*", "", regex=True) +df["data_set_path"] = df["data_set_type"].str.replace(".", "/", 1) +df["url"] = "https://seoflow.wyo.gov/Data/DataSet/Summary/Location/" + df["location_id"] + "/DataSet/" + df["data_set_path"] +df["provider_code"] = "wyseo" +df["parameter_name"] = df["data_set_type"] +df["parameter_group"] = "discharge" +df["parameter_id"] = df["provider_code"] + "-" + df["parameter_group"] +df["about_uri"] = df["about_url"] +df["name"] = df["Data_Set_Id"] + + + +df.head() + +# Print the first five rows of the "data_set_url" column +pd.options.display.max_colwidth = 300 +print(df["url"].head()) + +#Perform final field calculations on pandas DataFrame + +end_df = df[["about_uri", "url", "name", "provider_code", "parameter_id", "parameter_name", "parameter_group"]] +end_df.head() + +#Export and write to new CSV + +end_df.to_csv("C:\\Users\\ewiggans\\Desktop\\GeoConnexMap\\WyomingGage\\WY_download_table.csv") + +import json +import requests +from shapely.geometry import shape, Point + +def get_comid_intersect(geom): + # Convert the input geom to GeoJSON using Shapely + point = Point(geom) + + # Convert the Point to GeoJSON + geom_geojson = shape(point).__geo_interface__ + + url = 'https://nhdpv2-census.internetofwater.app/collections/2020/items?filter-lang=cql-json' + filter_ = { + 'intersects': [ + {'property': 'shape'}, + geom_geojson # Use the Shapely-converted GeoJSON + ] + } + headers = { + 'Content-Type': 'application/query-cql-json' + } + r = requests.post(url, headers=headers, json=filter_) + fc = r.json() + if 'features' in fc and len(fc['features']) > 0: + feature = fc['features'][0] + return feature['properties']['featureid'] + else: + return None + +# Update Cursor +with arcpy.da.UpdateCursor(WY_gage, ["Shape", "comid"]) as cursor: + for row in cursor: + geom = row[0] + comid = get_comid_intersect(geom) + if comid is not None: + print(comid, end='\r', flush=True) + row[1] = comid + cursor.updateRow(row) + +# Complete +print("comid calculated") + +# Read in the CSV file as a geodatabase table +csv_table = r"C:\Users\ewiggans\Desktop\GeoConnexMap\WyomingGage\nhdpv2_lookup.csv" +csv_table_name = "NHDPV2_Lookup" +arcpy.TableToTable_conversion(csv_table, arcpy.env.workspace, csv_table_name) + +# Add a new field 'comid_text' to the CSV table with a data type of TEXT +arcpy.AddField_management(csv_table_name, "comid_text", "TEXT") + +# Calculate the 'comid_text' field by copying the values from the 'comid' field +expression = "!comid!" +codeblock = "" +arcpy.CalculateField_management(csv_table_name, "comid_text", expression, "PYTHON3", codeblock) + +print("Added 'comid_text' field and set its values equal to 'comid' as text.") + + +# Join the CSV table to the "WyomingStreamGage" feature class based on the "comid" field +arcpy.AddJoin_management(WY_gage, "comid", csv_table_name, "comid_text", "KEEP_COMMON") + +# Calculate the "mainstem_uri" field to be equal to the "uri" field in the CSV table +expression = "!{}.uri!".format(csv_table_name) +arcpy.CalculateField_management(WY_gage, "mainstem_uri", expression, "PYTHON3") + +# Remove the join to the CSV table +arcpy.RemoveJoin_management(WY_gage, csv_table_name) + + +# Define the output GeoJSON file path +output_geojson_file = r"C:\Users\ewiggans\Desktop\GeoConnexMap\WyomingGage\WY_gage_export.geojson" + +# Export the feature class to GeoJSON +arcpy.FeaturesToJSON_conversion(in_features = WY_gage, + out_json_file = output_geojson_file, + geoJSON = "GEOJSON") + + +print(f"Exported {WY_gage} to {output_geojson_file} in GeoJSON format.") diff --git a/pygeoapi/features.config.yml b/pygeoapi/features.config.yml index 533a97d..c79e3f1 100644 --- a/pygeoapi/features.config.yml +++ b/pygeoapi/features.config.yml @@ -145,3 +145,25 @@ resources: name: ESRI data: https://geo.epa.ohio.gov/arcgis/rest/services/SurfaceWater/NPDES/MapServer/0 id_field: OHIO_EPA_NO + + wy/gages: + type: collection + title: Wyoming Gages + description: Gages from Wyoming State Engineers Office + keywords: + - wyoming + - gage + links: + - <<: *data-source + href: https://seoflow.wyo.gov/ + extents: + spatial: + bbox: [-111.06,40.99,-104.05,45.01] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: null + end: null + providers: + - <<: *provider + table: wy_gages + id_field: id