diff --git a/HISTORY.rst b/HISTORY.rst index a9195afa..2e800f8c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,10 @@ DeepForest Change Log ===================== +**1.3.5** + +Create a utilities.download_ArcGIS_REST function to download tiles from ArcGIS REST API. For example to download NAIP imagery. + **1.3.3** * Allow for annotations_file to be none in split_raster, for use in data preprocessing. diff --git a/deepforest/main.py b/deepforest/main.py index d50e8f3f..a7238f38 100644 --- a/deepforest/main.py +++ b/deepforest/main.py @@ -205,17 +205,17 @@ def create_trainer(self, logger=None, callbacks=[], **kwargs): enable_checkpointing = True else: enable_checkpointing = False - + trainer_args = { - "logger":logger, - "max_epochs":self.config["train"]["epochs"], - "enable_checkpointing":enable_checkpointing, - "devices":self.config["devices"], - "accelerator":self.config["accelerator"], - "fast_dev_run":self.config["train"]["fast_dev_run"], - "callbacks":callbacks, - "limit_val_batches":limit_val_batches, - "num_sanity_val_steps":num_sanity_val_steps + "logger": logger, + "max_epochs": self.config["train"]["epochs"], + "enable_checkpointing": enable_checkpointing, + "devices": self.config["devices"], + "accelerator": self.config["accelerator"], + "fast_dev_run": self.config["train"]["fast_dev_run"], + "callbacks": callbacks, + "limit_val_batches": limit_val_batches, + "num_sanity_val_steps": num_sanity_val_steps } # Update with kwargs to allow them to override config trainer_args.update(kwargs) diff --git a/deepforest/utilities.py b/deepforest/utilities.py index 5afca4a8..ba34fccd 100644 --- a/deepforest/utilities.py +++ b/deepforest/utilities.py @@ -9,6 +9,8 @@ import numpy as np import pandas as pd import rasterio +from rasterio.crs import CRS +import requests import shapely import xmltodict import yaml @@ -560,3 +562,73 @@ def project_boxes(df, root_dir, transform=True): df.crs = crs return df + + +def download_ArcGIS_REST(url, + xmin, + ymin, + xmax, + ymax, + savedir, + additional_params=None, + image_name="image.tiff", + download_service="exportImage"): + """ + Fetch data from a web server using geographic boundaries. The data is saved as a GeoTIFF file. The bbox is in the format of xmin, ymin, xmax, ymax for lat long coordinates.. + This function is used to download data from an ArcGIS REST service, not WMTS or WMS services. + Example url: https://gis.calgary.ca/arcgis/rest/services/pub_Orthophotos/CurrentOrthophoto/ImageServer/ + + There are many types of ImageServer, mapServer and open source alternatives. This function is designed to work with ArcGIS ImageServer, but may work with other services. Its very hard to anticipate all params and add to additional_params and download_service name to meet the specifications. + Parameters: + - url: The base URL of the web server. + - xmin: The minimum x-coordinate (longitude). + - ymin: The minimum y-coordinate (latitude). + - xmax: The maximum x-coordinate (longitude). + - ymax: The maximum y-coordinate (latitude). + - savedir: The directory to save the downloaded image. + - additional_params: Additional query parameters to include in the request. + + Returns: + The response from the web server. + """ + # Construct the query parameters with the geographic boundaries + params = {"f": "json"} + + # Make the GET request with the URL and parameters + response = requests.get(url, params=params) + + # turn into dict + response_dict = json.loads(response.content) + spatialReference = response_dict["spatialReference"] + if "latestWkid" in spatialReference: + wkid = spatialReference["latestWkid"] + crs = CRS.from_epsg(wkid) + elif 'wkt' in spatialReference: + crs = CRS.from_wkt(spatialReference['wkt']) + + # Convert bbox into image coordinates + bbox = f"{xmin},{ymin},{xmax},{ymax}" + bounds = gpd.GeoDataFrame(geometry=[shapely.geometry.box(ymin, xmin, ymax, xmax)], + crs='EPSG:4326').to_crs(crs).bounds + + # update the params + params.update({ + "bbox": f"{bounds.minx[0]},{bounds.miny[0]},{bounds.maxx[0]},{bounds.maxy[0]}", + "f": "image", + 'format': 'tiff', + }) + + # add any additional parameters + if additional_params: + params.update(additional_params) + + # Make the GET request with the URL and parameters + download_url_service = f"{url}/{download_service}" + response = requests.get(download_url_service, params=params) + if response.status_code == 200: + filename = f"{savedir}/{image_name}" + with open(filename, "wb") as f: + f.write(response.content) + return filename + else: + raise Exception(f"Failed to fetch data: {response.code}") diff --git a/docs/annotation.md b/docs/annotation.md index d3780927..1ede7081 100644 --- a/docs/annotation.md +++ b/docs/annotation.md @@ -117,4 +117,35 @@ Many projects have a linear concept of annotations with all the annotations coll # Please consider making your annotations open-source! -The DeepForest backbone tree and bird models are not perfect. Please consider posting any annotations you make on zenodo, or sharing them with DeepForest mantainers. Open an [issue](https://github.com/weecology/DeepForest/issues) and tell us about the RGB data and annotations. For example, we are collecting tree annotations to create an [open-source benchmark](https://milliontrees.idtrees.org/). Please consider sharing data to make the models stronger and benefit you and other users. \ No newline at end of file +The DeepForest backbone tree and bird models are not perfect. Please consider posting any annotations you make on zenodo, or sharing them with DeepForest mantainers. Open an [issue](https://github.com/weecology/DeepForest/issues) and tell us about the RGB data and annotations. For example, we are collecting tree annotations to create an [open-source benchmark](https://milliontrees.idtrees.org/). Please consider sharing data to make the models stronger and benefit you and other users. + +# How can I get new airborne data? + +Many remote sensing assets are stored as an ImageServer within ArcGIS REST protocol. As part of automating airborne image workflows, we have tools that help work with these assets. For example [California NAIP data](https://map.dfg.ca.gov/arcgis/rest/services/Base_Remote_Sensing/NAIP_2020_CIR/ImageServer). + +More work is needed to encompass the *many* different param settings and specifications. We welcome pull requests from those with experience with [WebMapTileServices](https://enterprise.arcgis.com/en/server/latest/publish-services/windows/wmts-services.htm). + +## Example: Specificy a lat long box and crop an ImageServer asset + +``` +from deepforest import utilities +import matplotlib.pyplot as plt +import rasterio as rio +import os + +url = "https://map.dfg.ca.gov/arcgis/rest/services/Base_Remote_Sensing/NAIP_2020_CIR/ImageServer/" +xmin, ymin, xmax, ymax = 40.49457, -124.112622, 40.493891, -124.111536 +tmpdir = +image_name = "example_crop.tif" +filename = utilities.download_ArcGIS_REST(url, xmin, ymin, xmax, ymax, savedir=tmpdir, image_name=image_name) + +# Check the saved file exists +assert os.path.exists("{}/{}".format(tmpdir, image_name)) + +# Confirm file has crs and show +with rio.open("{}/{}".format(tmpdir, image_name)) as src: + assert src.crs is not None + # Show + plt.imshow(src.read().transpose(1,2,0)) + plt.show() +``` \ No newline at end of file diff --git a/docs/landing.md b/docs/landing.md index 8e791585..17d6a15c 100644 --- a/docs/landing.md +++ b/docs/landing.md @@ -1,6 +1,6 @@ # What is DeepForest? -DeepForest is a python package for training and predicting ecological objects in airborne imagery. DeepForest currently comes with a tree crown object detection model and a bird detection model. Both are single class modules that can be extended to species classification based on new data. Users can extend these models by annotating and training custom models. +DeepForest is a python package for training and predicting ecological objects in airborne imagery. DeepForest currently comes with a tree crown object detection model and a bird detection model. Both are single class modules that can be extended to species classification based on new data. Users can extend these models by annotating and training custom models. ![](../www/image.png) diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 3f80332c..832ea07d 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -13,6 +13,10 @@ #import general model fixture from .conftest import download_release +import requests +import pytest +import matplotlib.pyplot as plt +import cv2 @pytest.fixture() def config(): @@ -139,4 +143,50 @@ def test_boxes_to_shapefile_unprojected(m, flip_y_axis, projected): # Confirm that each boxes within image bounds geom = geometry.box(*r.bounds) - assert all(gdf.geometry.apply(lambda x: geom.intersects(geom)).values) \ No newline at end of file + assert all(gdf.geometry.apply(lambda x: geom.intersects(geom)).values) + +def url(): + return [ + "https://map.dfg.ca.gov/arcgis/rest/services/Base_Remote_Sensing/NAIP_2020_CIR/ImageServer/", + "https://gis.calgary.ca/arcgis/rest/services/pub_Orthophotos/CurrentOrthophoto/ImageServer/", + "https://orthos.its.ny.gov/arcgis/rest/services/wms/Latest/MapServer" + ] + +def boxes(): + return [ + (40.49457, -124.112622, 40.493891, -124.111536), + (51.07332, -114.12529, 51.072134, -114.12117), + (41.111626,-73.763941, 41.111032,-73.763447) + ] + +def additional_params(): + return [ + None, + None, + {"format":"png"} + ] + +def download_service(): + return [ + "exportImage", + "exportImage", + "export" + ] + +# Pair each URL with its corresponding box +url_box_pairs = list(zip(["CA.tif","MA.tif","NY.png"],url(), boxes(), additional_params(), download_service())) +@pytest.mark.parametrize("image_name, url, box, params,download_service_name", url_box_pairs) +def test_download_ArcGIS_REST(tmpdir, image_name, url, box, params, download_service_name): + xmin, ymin, xmax, ymax = box + # Call the function + filename = utilities.download_ArcGIS_REST(url, xmin, ymin, xmax, ymax, savedir=tmpdir, image_name=image_name,additional_params=params, download_service=download_service_name) + # Check the saved file + assert os.path.exists("{}/{}".format(tmpdir, image_name)) + # Confirm file has crs + with rio.open(filename) as src: + if image_name.endswith('.tif'): + assert src.crs is not None + plt.imshow(src.read().transpose(1,2,0)) + else: + assert src.crs is None + plt.imshow(cv2.imread(filename)[:,:,::-1])