diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 892d845..71aa90f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,6 +20,9 @@ jobs: with: postgresql password: ${{ env.POSTGRESQL_PASSWORD }} postgresql db: "test" + - name: Run pygeoapi + run: | + docker run -d -it --rm -p 5000:80 $(docker build -q .) - name: Install requirements 📦 run: | pip3 install https://github.com/geopython/pygeoapi/archive/refs/heads/master.zip @@ -33,9 +36,11 @@ jobs: env: POSTGRESQL_PASSWORD: ${{ env.POSTGRESQL_PASSWORD }} run: | - pytest tests/test_ckan.py + pytest tests/test_ckan_provider.py pytest tests/test_postgresql_provider.py - pytest tests/test_sparql.py + pytest tests/test_sitemap_process.py + pytest tests/test_sparql_provider.py + pytest tests/test_xml_formatter.py - name: run flake8 ⚙️ run: | find . -type f -name "*.py" | xargs flake8 diff --git a/Dockerfile b/Dockerfile index 4f17fe5..cc6c0b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM geopython/pygeoapi:latest +ADD ./docker/pygeoapi.config.yml /pygeoapi/local.config.yml + ADD . /pygeoapi_plugins RUN pip3 install -e /pygeoapi_plugins diff --git a/README.md b/README.md index 25ba7e8..ce1ca96 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ pygeoapi plugins developed by the Center for Geospatial Solutions ## OGC API - Features -CGS additional feature providers are listed below, along with a matrix of supported query parameters. +Additional OGC API - Feature providers are listed below, along with a matrix of supported query parameters. | Provider | Property Filters/Display | Result Type | BBox | Datetime | Sort By | Skip Geometry | CQL | Transactions | CRS | | ------------------ | ------------------------ | ------------ | ---- | -------- | ------- | ------------- | --- | ------------ | --- | @@ -80,7 +80,11 @@ The SPARQL Provider only uses variables prefixed with sparql\_ in the configurat ## OGC API - Processes -CGS provides an intersection process, using OGC API - Features Part 3: Filtering to return CQL intersections of features. +Additional OGC API - Process are listed below + +### Intersector + +The intersection process uses OGC API - Features Part 3: Filtering to return CQL intersections of features. An example configuration in a pygeoapi configuration is below. ``` @@ -90,4 +94,17 @@ An example configuration in a pygeoapi configuration is below. name: pygeoapi_plugins.process.intersect.IntersectionProcessor ``` -This plugin is used in https://nhdpv2-census.internetofwater.app/. +This plugin is used in https:/reference.geoconnex.us/. + +### Sitemap Generator + +The Sitemap Generator process makes use of the XML formatter and OGC API - Features to generate a sitemap of the pygeoapi instance. +This can be used with the python package [sitemap-generator](https://github.com/cgs-earth/sitemap-generator) to generate a sitemap index. +An example configuration in a pygeoapi configuration is below. + +``` + sitemap-generator: + type: process + processor: + name: pygeoapi_plugins.process.sitemap.SitemapProcessor +``` diff --git a/docker/examples/sparql/sparql.config.yml b/docker/examples/sparql/sparql.config.yml index 05c733a..cb34871 100644 --- a/docker/examples/sparql/sparql.config.yml +++ b/docker/examples/sparql/sparql.config.yml @@ -184,8 +184,3 @@ resources: homepage: foaf:homepage wikipedia_link: foaf:isPrimaryTopicOf time_zone: dbp:timezone - - # intersector: - # type: process - # processor: - # name: pygeoapi_plugins.process.intersect.IntersectionProcessor diff --git a/docker/pygeoapi.config.yml b/docker/pygeoapi.config.yml new file mode 100644 index 0000000..7d3bae6 --- /dev/null +++ b/docker/pygeoapi.config.yml @@ -0,0 +1,206 @@ +server: + bind: + host: 0.0.0.0 + port: 80 + url: http://localhost:5000 + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + gzip: false + language: en-US + cors: true + pretty_print: true + limit: 10 + # templates: /path/to/templates + map: + url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png + attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + ogc_schemas_location: /schemas.opengis.net + +logging: + level: ERROR + #logfile: /tmp/pygeoapi.log + +metadata: + identification: + title: pygeoapi Demo instance - running latest GitHub version + description: pygeoapi provides an API to geospatial data + keywords: + - geospatial + - data + - api + keywords_type: theme + terms_of_service: https://creativecommons.org/licenses/by/4.0/ + url: https://github.com/geopython/pygeoapi + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: pygeoapi Development Team + url: https://pygeoapi.io + contact: + name: Kralidis, Tom + position: Lead Dev + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Canada + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Hours of Service + instructions: During hours of service. Off on weekends. + role: pointOfContact + +resources: + places: + type: collection + title: Places + description: Cities around the world and their DBpedia context + keywords: + - sparql + - pygeoapi + - rdf + context: + - name: schema:name + description: schema:description + subjectOf: schema:subjectOf + links: + - type: application/html + rel: canonical + title: data source + href: http://dbpedia.org/ + hreflang: en-US + extents: + spatial: + bbox: [-180, -90, 180, 90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: null + end: null + providers: + - type: feature + name: pygeoapi_plugins.provider.sparql.SPARQLProvider + data: /pygeoapi_plugins/tests/data/places.csv + id_field: index + geometry: + x_field: lon + y_field: lat + sparql_provider: CSV + sparql_endpoint: https://dbpedia.org/sparql + sparql_subject: uri + sparql_predicates: + population: dbo:populationTotal + country: + leader: dbpedia2:leaderName + populated: + type: collection + title: Populated Places + description: Populated places, public domain and their DBpedia context + keywords: + - sparql + - pygeoapi + - rdf + links: + - type: text/html + rel: canonical + title: information + href: http://www.naturalearthdata.com/ + hreflang: en-US + extents: + spatial: + bbox: [-180, -90, 180, 90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2011-11-11 + end: null # or empty (either means open ended) + providers: + - type: feature + name: pygeoapi_plugins.provider.sparql.SPARQLProvider + data: /pygeoapi_plugins/tests/data/ne_110m_populated_places_simple.geojson + id_field: id + sparql_provider: GeoJSON + sparql_endpoint: https://dbpedia.org/sparql + sparql_subject: uri + sparql_predicates: + leader: dbpedia2:leaderName|dbp:leaderName + population: dbo:populationTotal|dbp:populationCensus + states: + type: collection + title: States + description: U.S. States and their DBpedia context + keywords: + - States + - Census + geojsonld: false + links: + - type: application/html + rel: canonical + title: data source + href: https://www.hydroshare.org/resource/3295a17b4cc24d34bd6a5c5aaf753c50/data/contents/states.gpkg + hreflang: en-US + extents: + spatial: + bbox: [-170, 15, -51, 72] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: null + end: null + providers: + - type: feature + name: pygeoapi_plugins.provider.sparql.SPARQLProvider + data: /pygeoapi_plugins/tests/data/states.gpkg + id_field: GEOID + table: states + sparql_provider: SQLiteGPKG + sparql_endpoint: https://dbpedia.org/sparql + sparql_subject: " :NAME" + sparql_predicates: + senator: dbp:senators + motto: dbo:motto|dbp:motto + capital: dbo:capital + homepage: foaf:homepage + wikipedia_link: foaf:isPrimaryTopicOf + time_zone: dbp:timezone + reservoirs: + type: collection + title: New Mexico Reservoirs + description: This is a point coverage of dams in the New Mexico, which originally was derived from the national inventory of dams data base (U.S. Army Corps of Engineers, 1982) + keywords: + - pygeoapi + - ckan + - api + context: + - name: schema:name + description: schema:description + subjectOf: schema:subjectOf + links: + - type: application/html + rel: canonical + title: data source + href: https://catalog.newmexicowaterdata.org/mn_MN/dataset/new-mexico-reservoirs/resource/08369d21-520b-439e-97e3-5ecb50737887 + hreflang: en-US + extents: + spatial: + bbox: [-109, 31, -103, 37] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: null + end: null + providers: + - type: feature + name: pygeoapi_plugins.provider.ckan.CKANProvider + data: https://catalog.newmexicowaterdata.org/api/3/action/datastore_search + resource_id: 08369d21-520b-439e-97e3-5ecb50737887 + id_field: _id + x_field: LONDD + y_field: LATDD + intersector: + type: process + processor: + name: pygeoapi_plugins.process.intersect.IntersectionProcessor + sitemap-generator: + type: process + processor: + name: pygeoapi_plugins.process.sitemap.SitemapProcessor diff --git a/pygeoapi_plugins/formatter/__init__.py b/pygeoapi_plugins/formatter/__init__.py new file mode 100644 index 0000000..45279d0 --- /dev/null +++ b/pygeoapi_plugins/formatter/__init__.py @@ -0,0 +1,30 @@ +# ================================================================= +# +# Author: Benjamin Webb +# +# Copyright (c) 2023 Center for Geospatial Solutions +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +"""Output formatter package""" diff --git a/pygeoapi_plugins/formatter/xml.py b/pygeoapi_plugins/formatter/xml.py new file mode 100644 index 0000000..c22b74f --- /dev/null +++ b/pygeoapi_plugins/formatter/xml.py @@ -0,0 +1,116 @@ +# ================================================================= +# +# Author: Benjamin Webb +# +# Copyright (c) 2023 Center for Geospatial Solutions +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +from datetime import datetime +import io +import logging +import xml.etree.ElementTree as ET + +from pygeoapi.formatter.base import ( + BaseFormatter, FormatterSerializationError) + +LOGGER = logging.getLogger(__name__) + +URLSET = ''' + + +''' +URLSET_FOREACH = ''' + + {} + {} + +''' + + +class XMLFormatter(BaseFormatter): + """XML formatter""" + + def __init__(self, formatter_def: dict): + """ + Initialize object + + :param formatter_def: formatter definition + + :returns: `pygeoapi.formatter.xml.XMLFormatter` + """ + + geom = False + self.uri_field = formatter_def.get('uri_field') + super().__init__({'name': 'xml', 'geom': geom}) + self.mimetype = 'application/xml; charset=utf-8' + + def write(self, options: dict = {}, data: dict = None) -> str: + """ + Generate data in XML format + + :param options: XML formatting options + :param data: dict of GeoJSON data + + :returns: string representation of format + """ + + try: + feature = list(data['features'][0]) + except IndexError: + LOGGER.error('no features') + return str() + + lastmod = datetime.now().strftime('%Y-%m-%dT%H:%M:%SZ') + root = ET.fromstring(URLSET) + tree = ET.ElementTree(root) + try: + ET.indent(tree) + except AttributeError: + LOGGER.warning('Unable to indent') + + try: + for i, feature in enumerate(data['features']): + if i >= 50000: + LOGGER.warning('Maximum size of sitemap reached') + break + + try: + loc = feature['properties'][self.uri_field] + except KeyError: + loc = feature['@id'] + + _ = URLSET_FOREACH.format(loc, lastmod) + root.append(ET.fromstring(_)) + + except ValueError as err: + LOGGER.error(err) + raise FormatterSerializationError('Error writing XML output') + + output = io.BytesIO() + tree.write(output, encoding='utf-8', xml_declaration=True) + return output.getvalue() + + def __repr__(self): + return f' {self.name}' diff --git a/pygeoapi_plugins/process/sitemap.py b/pygeoapi_plugins/process/sitemap.py new file mode 100644 index 0000000..7cb8507 --- /dev/null +++ b/pygeoapi_plugins/process/sitemap.py @@ -0,0 +1,255 @@ +# ================================================================= +# +# Author: Benjamin Webb +# +# Copyright (c) 2023 Center for Geospatial Solutions +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import io +import math +import os +import logging +import zipfile + +from pygeoapi.plugin import load_plugin +from pygeoapi.process.base import BaseProcessor +from pygeoapi.linked_data import geojson2jsonld +from pygeoapi.openapi import get_oas +from pygeoapi.util import (yaml_load, get_provider_default, url_join, + filter_dict_by_key_value) + +from pygeoapi_plugins.formatter.xml import XMLFormatter + + +LOGGER = logging.getLogger(__name__) + +with open(os.getenv('PYGEOAPI_CONFIG'), encoding='utf8') as fh: + CONFIG = yaml_load(fh) + COLLECTIONS = filter_dict_by_key_value(CONFIG['resources'], + 'type', 'collection') + # TODO: Filter collections for those that support CQL + + +PROCESS_DEF = CONFIG['resources']['sitemap-generator'] +PROCESS_DEF.update({ + 'version': '0.1.0', + 'id': 'sitemap-generator', + 'title': 'Sitemap Generator', + 'description': ('A process that returns a sitemap of' + 'all pygeoapi endpoints.'), + 'links': [{ + 'type': 'text/html', + 'rel': 'about', + 'title': 'information', + 'href': 'https://developers.google.com/search/docs/crawling-indexing/sitemaps/overview', # noqa + 'hreflang': 'en-US' + }], + 'inputs': { + 'include-common': { + 'title': { + 'en': 'Include OGC API - Common' + }, + 'description': { + 'en': 'Boolean value controlling the generation of a sitemap ' + 'for OGC API - Common endpoints' + }, + 'keywords': { + 'en': ['sitemap', 'ogc', 'OGC API - Common', 'pygeoapi'] + }, + 'schema': { + 'type': 'boolean', + 'default': True + }, + 'minOccurs': 0, + 'maxOccurs': 1, + 'metadata': None, # TODO how to use? + }, + 'include-features': { + 'title': { + 'en': 'Include OGC API - Features' + }, + 'description': { + 'en': 'Boolean value controlling the generation of a sitemap ' + 'for individual OGC API - Features endpoints' + }, + 'keywords': { + 'en': ['sitemap', 'ogc', 'OGC API - Features', 'pygeoapi'] + }, + 'schema': { + 'type': 'boolean', + 'default': True + }, + 'minOccurs': 0, + 'maxOccurs': 1, + 'metadata': None, # TODO how to use? + }, + 'zip': { + 'title': { + 'en': 'ZIP response' + }, + 'description': { + 'en': 'Boolean whether to ZIP the response' + }, + 'keywords': { + 'en': ['sitemap', 'zip', 'pygeoapi'] + }, + 'schema': { + 'type': 'boolean', + 'default': False + }, + 'minOccurs': 0, + 'maxOccurs': 1, + 'metadata': None, # TODO how to use? + }, + }, + 'outputs': { + 'common.xml': { + 'title': { + 'en': 'OGC API - Common Sitemap' + }, + 'description': { + 'en': 'A sitemap of the OGC API - Common end points for the ' + 'pygeoapi instance.' + }, + 'schema': { + 'type': 'object', + 'contentMediaType': 'application/json' + } + }, + 'sitemap.zip': { + 'title': { + 'en': 'Sitemap' + }, + 'description': { + 'en': 'A sitemap of the pygeoapi instance' + }, + 'schema': { + 'type': 'object', + 'contentMediaType': 'application/zip' + } + } + }, + 'example': { + 'inputs': { + 'include-features': False + } + } +}) + + +class SitemapProcessor(BaseProcessor): + """Sitemap Processor""" + + def __init__(self, processor_def): + """ + Initialize object + + :param processor_def: provider definition + + :returns: pygeoapi.process.sitemap.SitemapProcessor + """ + LOGGER.debug('SitemapProcesser init') + super().__init__(processor_def, PROCESS_DEF) + self.config = CONFIG + self.base_url = self.config['server']['url'] + self.xml = XMLFormatter({}) + + def execute(self, data): + """ + Execute Sitemap Process + + :param data: processor arguments + + :returns: 'application/json' + """ + mimetype = 'application/json' + common = data.get('include-common', True) + features = data.get('include-features', True) + if data.get('zip'): + LOGGER.debug('Returning zipped response') + zip_output = io.BytesIO() + with zipfile.ZipFile(zip_output, 'w') as zipf: + for filename, content in self.generate(common, features): + zipf.writestr(filename, content) + return 'application/zip', zip_output.getvalue() + + else: + LOGGER.debug('Returning response') + return mimetype, dict(self.generate(common, features)) + + def generate(self, include_common, include_features): + """ + Execute Sitemap Process + + :param include_common: Include OGC API - Common endpoints + :param include_features: Include OGC API - Features endpoints + + :returns: 'application/json' + """ + if include_common: + LOGGER.debug('Generating common.xml') + oas = {'features': []} + for path in get_oas(self.config).get('paths'): + if r'{jobId}' not in path and r'{featureId}' not in path: + path_uri = url_join(self.base_url, path) + oas['features'].append({'@id': path_uri}) + yield ('common.xml', self.xml.write(data=oas)) + + if include_features: + LOGGER.debug('Generating collections sitemap') + for name, c in COLLECTIONS.items(): + LOGGER.debug(f'Generating sitemap(s) for {name}') + p = get_provider_default(c['providers']) + provider = load_plugin('provider', p) + hits = provider.query(resulttype='hits').get('numberMatched') + iterations = range(math.ceil(hits / 50000)) + for i in iterations: + yield (f'{name}__{i}.xml', + self._generate(i, name, provider)) + + def _generate(self, index, dataset, provider, n=50000): + """ + Private Function: Generate sitemap + + :param index: feature list index + :param dataset: OGC API Provider name + :param provider: OGC API Provider definition + :param n: Number per index + + :returns: List of GeoJSON Features + """ + + content = provider.query(offset=(n*index), limit=n) + content['links'] = [] + content = geojson2jsonld( + self, content, dataset, id_field=(provider.uri_field or 'id') + ) + return self.xml.write(data=content) + + def get_collections_url(self): + return url_join(self.base_url, 'collections') + + def __repr__(self): + return f' {self.name}' diff --git a/tests/test_ckan.py b/tests/test_ckan_provider.py similarity index 100% rename from tests/test_ckan.py rename to tests/test_ckan_provider.py diff --git a/tests/test_sitemap_process.py b/tests/test_sitemap_process.py new file mode 100644 index 0000000..fad591d --- /dev/null +++ b/tests/test_sitemap_process.py @@ -0,0 +1,97 @@ +# ================================================================= +# +# Author: Benjamin Webb +# +# Copyright (c) 2023 Center for Geospatial Solutions +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import io +import pytest +from requests import Session +import xml +import zipfile + +from pygeoapi.util import url_join + +PYGEOAPI_URL = 'http://localhost:5000' +PROCESS_URL = url_join(PYGEOAPI_URL, 'processes/sitemap-generator/execution') +HTTP = Session() + + +@pytest.fixture +def body(): + return { + 'inputs': { + 'include-common': True, + 'include-features': False, + 'zip': False + } + } + + +def test_sitemap_generator(body): + body['inputs']['include-features'] = True + r = HTTP.post(PROCESS_URL, json=body) + assert r.status_code == 200 + + sitemap = r.json() + assert len(sitemap) == 5 + + common = sitemap.pop('common.xml') + assert len(common) == 2279 + + root = xml.etree.ElementTree.fromstring(common) + assert all(i.tag == j.tag for (i, j) in zip(root, root.findall('url'))) + + assert all(f.endswith('__0.xml') for f in sitemap) + + +def test_sitemap_no_common(body): + body['inputs']['include-common'] = False + r = HTTP.post(PROCESS_URL, json=body) + assert r.status_code == 200 + + sitemap = r.json() + assert len(sitemap) == 0 + + +def test_sitemap_no_features(body): + r = HTTP.post(PROCESS_URL, json=body) + assert r.status_code == 200 + + sitemap = r.json() + assert len(sitemap) == 1 + + common = sitemap.pop('common.xml') + assert len(common) == 2279 + + +def test_sitemap_zip(body): + body['inputs']['zip'] = True + r = HTTP.post(PROCESS_URL, json=body) + assert r.status_code == 200 + + z = zipfile.ZipFile(io.BytesIO(r.content)) + assert len(z.namelist()) == 1 diff --git a/tests/test_sparql.py b/tests/test_sparql_provider.py similarity index 100% rename from tests/test_sparql.py rename to tests/test_sparql_provider.py diff --git a/tests/test_xml_formatter.py b/tests/test_xml_formatter.py new file mode 100644 index 0000000..8a3745d --- /dev/null +++ b/tests/test_xml_formatter.py @@ -0,0 +1,73 @@ +# ================================================================= +# +# Author: Benjamin Webb +# +# Copyright (c) 2023 Center for Geospatial Solutions +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +from datetime import datetime +import pytest +import xml.etree.ElementTree as ET + +from pygeoapi.provider.csv_ import CSVProvider + +from pygeoapi_plugins.formatter.xml import XMLFormatter + + +@pytest.fixture() +def config(): + return { + 'name': 'CSV', + 'type': 'feature', + 'data': 'tests/data/places.csv', + 'id_field': 'index', + 'uri_field': 'uri', + 'geometry': { + 'x_field': 'lon', + 'y_field': 'lat' + } + } + + +def test_xml_formatter(config): + p = CSVProvider(config) + f = XMLFormatter(config) + fc = p.query() + f_xml = f.write(data=fc) + + assert f.mimetype == 'application/xml; charset=utf-8' + + root = ET.fromstring(f_xml) + assert all(i.tag == j.tag for (i, j) in zip(root, root.findall('url'))) + + node = root.find('url') + assert node.find('loc').text == 'http://dbpedia.org/resource/Berlin' + + lastmod = node.find('lastmod').text + strptime = datetime.strptime(lastmod, '%Y-%m-%dT%H:%M:%SZ') + assert isinstance(strptime, datetime) + + now = datetime.now().strftime('%Y-%m-%dT%H:%M') + assert now in lastmod