Merge pull request #687 from weecology/download_webserver

Create a function to download tiles from ArcGIS ImageServer

HISTORY.rst

@@ -2,6 +2,10 @@
 DeepForest Change Log
 =====================
 
+**1.3.5**
+
+Create a utilities.download_ArcGIS_REST function to download tiles from ArcGIS REST API. For example to download NAIP imagery.
+
 **1.3.3**
 
 * Allow for annotations_file to be none in split_raster, for use in data preprocessing.

deepforest/main.py

@@ -205,17 +205,17 @@ def create_trainer(self, logger=None, callbacks=[], **kwargs):
             enable_checkpointing = True
         else:
             enable_checkpointing = False
-        
+
         trainer_args = {
-            "logger":logger,
-            "max_epochs":self.config["train"]["epochs"],
-            "enable_checkpointing":enable_checkpointing,
-            "devices":self.config["devices"],
-            "accelerator":self.config["accelerator"],
-            "fast_dev_run":self.config["train"]["fast_dev_run"],
-            "callbacks":callbacks,
-            "limit_val_batches":limit_val_batches,
-            "num_sanity_val_steps":num_sanity_val_steps
+            "logger": logger,
+            "max_epochs": self.config["train"]["epochs"],
+            "enable_checkpointing": enable_checkpointing,
+            "devices": self.config["devices"],
+            "accelerator": self.config["accelerator"],
+            "fast_dev_run": self.config["train"]["fast_dev_run"],
+            "callbacks": callbacks,
+            "limit_val_batches": limit_val_batches,
+            "num_sanity_val_steps": num_sanity_val_steps
         }
         # Update with kwargs to allow them to override config
         trainer_args.update(kwargs)

deepforest/utilities.py

@@ -9,6 +9,8 @@
 import numpy as np
 import pandas as pd
 import rasterio
+from rasterio.crs import CRS
+import requests
 import shapely
 import xmltodict
 import yaml
@@ -560,3 +562,73 @@ def project_boxes(df, root_dir, transform=True):
     df.crs = crs
 
     return df
+
+
+def download_ArcGIS_REST(url,
+                         xmin,
+                         ymin,
+                         xmax,
+                         ymax,
+                         savedir,
+                         additional_params=None,
+                         image_name="image.tiff",
+                         download_service="exportImage"):
+    """
+    Fetch data from a web server using geographic boundaries. The data is saved as a GeoTIFF file. The bbox is in the format of xmin, ymin, xmax, ymax for lat long coordinates..
+    This function is used to download data from an ArcGIS REST service, not WMTS or WMS services. 
+    Example url: https://gis.calgary.ca/arcgis/rest/services/pub_Orthophotos/CurrentOrthophoto/ImageServer/
+
+    There are many types of ImageServer, mapServer and open source alternatives. This function is designed to work with ArcGIS ImageServer, but may work with other services. Its very hard to anticipate all params and add to additional_params and download_service name to meet the specifications.
+    Parameters:
+    - url: The base URL of the web server.
+    - xmin: The minimum x-coordinate (longitude).
+    - ymin: The minimum y-coordinate (latitude).
+    - xmax: The maximum x-coordinate (longitude).
+    - ymax: The maximum y-coordinate (latitude).
+    - savedir: The directory to save the downloaded image.
+    - additional_params: Additional query parameters to include in the request.
+
+    Returns:
+    The response from the web server.
+    """
+    # Construct the query parameters with the geographic boundaries
+    params = {"f": "json"}
+
+    # Make the GET request with the URL and parameters
+    response = requests.get(url, params=params)
+
+    # turn into dict
+    response_dict = json.loads(response.content)
+    spatialReference = response_dict["spatialReference"]
+    if "latestWkid" in spatialReference:
+        wkid = spatialReference["latestWkid"]
+        crs = CRS.from_epsg(wkid)
+    elif 'wkt' in spatialReference:
+        crs = CRS.from_wkt(spatialReference['wkt'])
+
+    # Convert bbox into image coordinates
+    bbox = f"{xmin},{ymin},{xmax},{ymax}"
+    bounds = gpd.GeoDataFrame(geometry=[shapely.geometry.box(ymin, xmin, ymax, xmax)],
+                              crs='EPSG:4326').to_crs(crs).bounds
+
+    # update the params
+    params.update({
+        "bbox": f"{bounds.minx[0]},{bounds.miny[0]},{bounds.maxx[0]},{bounds.maxy[0]}",
+        "f": "image",
+        'format': 'tiff',
+    })
+
+    # add any additional parameters
+    if additional_params:
+        params.update(additional_params)
+
+    # Make the GET request with the URL and parameters
+    download_url_service = f"{url}/{download_service}"
+    response = requests.get(download_url_service, params=params)
+    if response.status_code == 200:
+        filename = f"{savedir}/{image_name}"
+        with open(filename, "wb") as f:
+            f.write(response.content)
+        return filename
+    else:
+        raise Exception(f"Failed to fetch data: {response.code}")

docs/annotation.md

@@ -117,4 +117,35 @@ Many projects have a linear concept of annotations with all the annotations coll
 
 # Please consider making your annotations open-source!
 
-The DeepForest backbone tree and bird models are not perfect. Please consider posting any annotations you make on zenodo, or sharing them with DeepForest mantainers. Open an [issue](https://github.com/weecology/DeepForest/issues) and tell us about the RGB data and annotations. For example, we are collecting tree annotations to create an [open-source benchmark](https://milliontrees.idtrees.org/). Please consider sharing data to make the models stronger and benefit you and other users. 
+The DeepForest backbone tree and bird models are not perfect. Please consider posting any annotations you make on zenodo, or sharing them with DeepForest mantainers. Open an [issue](https://github.com/weecology/DeepForest/issues) and tell us about the RGB data and annotations. For example, we are collecting tree annotations to create an [open-source benchmark](https://milliontrees.idtrees.org/). Please consider sharing data to make the models stronger and benefit you and other users. 
+
+# How can I get new airborne data?
+
+Many remote sensing assets are stored as an ImageServer within ArcGIS REST protocol. As part of automating airborne image workflows, we have tools that help work with these assets. For example [California NAIP data](https://map.dfg.ca.gov/arcgis/rest/services/Base_Remote_Sensing/NAIP_2020_CIR/ImageServer). 
+
+More work is needed to encompass the *many* different param settings and specifications. We welcome pull requests from those with experience with  [WebMapTileServices](https://enterprise.arcgis.com/en/server/latest/publish-services/windows/wmts-services.htm).
+
+## Example: Specificy a lat long box and crop an ImageServer asset 
+
+```
+from deepforest import utilities
+import matplotlib.pyplot as plt
+import rasterio as rio
+import os
+
+url = "https://map.dfg.ca.gov/arcgis/rest/services/Base_Remote_Sensing/NAIP_2020_CIR/ImageServer/"
+xmin, ymin, xmax, ymax = 40.49457, -124.112622, 40.493891, -124.111536
+tmpdir = <download_location>
+image_name = "example_crop.tif"
+filename = utilities.download_ArcGIS_REST(url, xmin, ymin, xmax, ymax, savedir=tmpdir, image_name=image_name)
+
+# Check the saved file exists
+assert os.path.exists("{}/{}".format(tmpdir, image_name))
+
+# Confirm file has crs and show
+with rio.open("{}/{}".format(tmpdir, image_name)) as src:
+    assert src.crs is not None
+    # Show
+    plt.imshow(src.read().transpose(1,2,0))
+    plt.show()
+```

docs/landing.md

@@ -1,6 +1,6 @@
 # What is DeepForest?
 
-DeepForest is a python package for training and predicting ecological objects in airborne imagery. DeepForest currently comes with a tree crown object detection model and a bird detection model. Both are single class modules that can be extended to species classification based on new data. Users can extend these models by annotating and training custom models.
+DeepForest is a python package for training and predicting ecological objects in airborne imagery. DeepForest currently comes with a tree crown object detection model and a bird detection model. Both are single class modules that can be extended to species classification based on new data. Users can extend these models by annotating and training custom models. 
 
 ![](../www/image.png)
 

tests/test_utilities.py

@@ -13,6 +13,10 @@
 
 #import general model fixture
 from .conftest import download_release
+import requests
+import pytest
+import matplotlib.pyplot as plt
+import cv2
 
 @pytest.fixture()
 def config():
@@ -139,4 +143,50 @@ def test_boxes_to_shapefile_unprojected(m, flip_y_axis, projected):
 
     # Confirm that each boxes within image bounds
     geom = geometry.box(*r.bounds)
-    assert all(gdf.geometry.apply(lambda x: geom.intersects(geom)).values)
+    assert all(gdf.geometry.apply(lambda x: geom.intersects(geom)).values)
+
+def url():
+    return [
+        "https://map.dfg.ca.gov/arcgis/rest/services/Base_Remote_Sensing/NAIP_2020_CIR/ImageServer/",
+        "https://gis.calgary.ca/arcgis/rest/services/pub_Orthophotos/CurrentOrthophoto/ImageServer/",
+        "https://orthos.its.ny.gov/arcgis/rest/services/wms/Latest/MapServer"
+    ]
+
+def boxes():
+    return [
+        (40.49457, -124.112622, 40.493891, -124.111536),
+        (51.07332, -114.12529, 51.072134, -114.12117),
+        (41.111626,-73.763941, 41.111032,-73.763447)
+    ]
+
+def additional_params():
+    return [
+        None,
+        None,
+        {"format":"png"}
+    ]
+
+def download_service():
+    return [
+        "exportImage",
+        "exportImage",
+        "export"
+    ]
+
+# Pair each URL with its corresponding box
+url_box_pairs = list(zip(["CA.tif","MA.tif","NY.png"],url(), boxes(), additional_params(), download_service()))
+@pytest.mark.parametrize("image_name, url, box, params,download_service_name", url_box_pairs)
+def test_download_ArcGIS_REST(tmpdir, image_name, url, box, params, download_service_name):
+    xmin, ymin, xmax, ymax = box
+    # Call the function
+    filename = utilities.download_ArcGIS_REST(url, xmin, ymin, xmax, ymax, savedir=tmpdir, image_name=image_name,additional_params=params, download_service=download_service_name)
+    # Check the saved file
+    assert os.path.exists("{}/{}".format(tmpdir, image_name))
+    # Confirm file has crs
+    with rio.open(filename) as src:
+        if image_name.endswith('.tif'):
+            assert src.crs is not None
+            plt.imshow(src.read().transpose(1,2,0))
+        else:
+            assert src.crs is None
+            plt.imshow(cv2.imread(filename)[:,:,::-1])