Merge pull request #46 from dan-bowerman/develop

Added support for recursive scraping of Esri and OGC services

run.py

@@ -13,6 +13,7 @@
 
 # FIXME clean this up
 app = Flask(__name__)
+
 reload(sys)
 sys.setdefaultencoding('utf8')
 app.config.from_object(config)

services/regparse/esri.py

@@ -229,6 +229,14 @@ def make_server_node(req):
     result = {}
     if 'scrape_only' in req:
         result['layerEntries'] = [{'index': index} for index in req['scrape_only']]
+    elif 'recursive' in req:
+        query_service = requests.get(req['service_url'] + "?f=pjson", proxies=flask.g.proxies)
+        service_json = query_service.json()
+        if service_json['type'] == 'Group Layer':
+            result['service_url'] = req['service_url'].rstrip('1234567890')
+            result['url'] = req['service_url'].rstrip('1234567890')
+        sublayer_json = service_json['subLayers']
+        result['layerEntries'] = [{'index': sl['id']} for sl in sublayer_json]
     else:
         result['layerEntries'] = []
     return result

services/regparse/ogc.py

@@ -1,3 +1,6 @@
+import requests
+from xml.dom import minidom
+
 """
 A WMS "parser" (barely does any parsing at the moment).
 """
@@ -65,6 +68,12 @@ def make_wms_node(req):
         result['legendMimeType'] = legend_format
     if 'scrape_only' in req:
         result['layerEntries'] = [{'id': id} for id in req['scrape_only']]
+    elif 'recursive' in req:
+        query_service = requests.get(req['service_url'] + '?SERVICE=WMS&REQUEST=GetCapabilities').content
+        xmldoc = minidom.parseString(query_service)
+        layers = xmldoc.getElementsByTagName('Layer')
+        result['layerEntries'] = [{"id": i.getElementsByTagName('Name')[0].firstChild.data,
+                                   "name": i.getElementsByTagName('Title')[0].firstChild.data} for i in layers]
     else:
         result['layerEntries'] = []
     return result

services/retrieval.py

@@ -1,4 +1,4 @@
-import db, json
+import db, json, re
 
 from utils import jsonp
 from flask import Response
@@ -92,3 +92,12 @@ class DocsV2(Docs):
     def __init__(self):
         super(DocsV2, self).__init__()
         self.version = '2'
+
+
+class Version(Doc):
+    def get(self):
+        with open("setup.py") as f:
+            for line in f:
+                if "version" in line:
+                    version_no = re.findall("\d+\.\d+.\d+", line)
+        return version_no

services/v2.py

@@ -1,6 +1,6 @@
-from registration import Register, Refresh
+from registration import Register, Refresh, Update
 from upgrade import Upgrade
-from retrieval import DocV2, DocsV2
+from retrieval import DocV2, DocsV2, Version
 from debug import AccessLog, Log, AllKeys
 
 from flask import Blueprint
@@ -18,6 +18,8 @@ def make_blueprint(app):
     api.add_resource(DocV2, '/doc/<string:lang>/<string:smallkey>')
     api.add_resource(DocsV2, '/docs/<string:lang>/<string:smallkeylist>')
     api.add_resource(Upgrade, '/upgrade/2.0/<string:key>')
+    api.add_resource(Update, '/update/2.0/<string:key>')
+    api.add_resource(Version, '/version/')
     if app.config.get('DEBUG_ENDPOINTS'):
         api.add_resource(AccessLog, '/accesslog', '/accesslog/<int:index>')
         api.add_resource(Log, '/log', '/log/<int:index>')

setup.py

@@ -7,7 +7,7 @@
 setup(
     name="rcs",
     description="RAMP Configuration Service",
-    version="2.0.0",
+    version="2.0.1",
     author="Environment and Climate Change Canada",
     author_email="[email protected]",
     url="https://github.com/fgpv-vpgf/rcs",

static/test.html

@@ -102,6 +102,8 @@
             <input type="button" class="btn btn-default samplejson" value="esriFeature (Class Breaks)" data-obj="sampleFeature2" />
             <input type="button" class="btn btn-default samplejson" value="esriFeature (Specific Sublayers)" data-obj="sampleScrapeOnly" />
             <input type="button" class="btn btn-default samplejson" value="esriFeature (w/ Metadata)" data-obj="sampleCatalogue" />
+            <input type="button" class="btn btn-default samplejson" value="esri Group Layer (recursive)" data-obj="sampleRecursive" />
+            <input type="button" class="btn btn-default samplejson" value="ogcWMS Group Layer (recursive)" data-obj="sampleRecursiveWMS" />
         </div>
     </div>
     <div class="row"><div id="log" class="form-group col-md-12">
@@ -225,7 +227,7 @@
         	},
         	"version": "2.0"
         },
-        sampleCatalogue =   {
+        sampleCatalogue = {
             "version": "2.0",
           	"fr": {
           		"service_url": "http://section917.cloudapp.net/arcgis/rest/services/TestData/ClassBreaks/MapServer/0",
@@ -245,6 +247,40 @@
                       "metadata_url": "https://gcgeo.gc.ca/geonetwork/srv/eng/xml.metadata.download?uuid=ce7873ff-fbc0-4864-946e-6a1b4d739154&fromWorkspace="
                   }
           	}
+        },
+        sampleRecursive = {
+          	"fr": {
+          		"service_url": "http://ec.gc.ca/arcgis/rest/services/CESI_FGP_All_Layers/MapServer/0",
+          		"recursive": true,
+          		"service_type": "esriMapServer",
+          		"service_name": "CESI: Protected areas, Canada"
+          	},
+          	"en": {
+          		"service_url": "http://ec.gc.ca/arcgis/rest/services/CESI_FGP_All_Layers/MapServer/0",
+          		"recursive": true,
+          		"service_type": "esriMapServer",
+          		"service_name": "CESI: Protected areas, Canada"
+          	},
+          	"version": "2.0"
+        },
+        sampleRecursiveWMS = {
+        	"fr": {
+        		"feature_info_format": "text/plain",
+        		"service_url": "http://maps.geogratis.gc.ca/wms/railway_en",
+        		"service_type": "ogcWms",
+        		"legend_format": "image/png",
+        		"recursive": true,
+        		"service_name": "WMS french"
+        	},
+        	"en": {
+        		"feature_info_format": "text/plain",
+        		"service_url": "http://maps.geogratis.gc.ca/wms/railway_en",
+        		"service_type": "ogcWms",
+        		"legend_format": "image/png",
+        		"recursive": true,
+        		"service_name": "WMS english"
+        	},
+        	"version": "2.0"
         };
 
     function get_versioned_url() {