Skip to content

Commit

Permalink
Fix SPARQL provider and CI tests (#26)
Browse files Browse the repository at this point in the history
* Update sparql.py

* Update additional pages from OACommon

* Update sparql.py

Update sparql.py

Update sparql.py

* Update test_sparql_provider.py

* Update sparql.py

Update sparql.py

Update sparql.py

Update sparql.py

Update sparql.py

Update sparql.py

Update sparql.py

* POST sparql queries
  • Loading branch information
webb-ben authored Sep 26, 2024
1 parent 7c5623e commit 1eb0430
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 16 deletions.
30 changes: 19 additions & 11 deletions pygeoapi_plugins/provider/sparql.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,9 @@ def __init__(self, provider_def):
# Set SPARQL query parameters
query = provider_def.get('sparql_query', {})
self.convert = query.get('convert', True)
self.sparql_endpoint = query.get('endpoint')
self.sparql = SPARQLWrapper(query.get('endpoint'))
self.sparql.setMethod('POST')
self.sparql.setReturnFormat(JSON)

select = query.get('select', '*')
self.select = _SELECT.format(select=select)
Expand Down Expand Up @@ -286,15 +288,15 @@ def _clean_result(self, result):
if _id not in ret:
ret[_id] = {k: [] for k in v.keys()}

# Iterate over each property-value pair for this binding
for k, v in v.items():
# Iterate over each property-value pair for this binding
for k, v_ in v.items():
# Ensure the property's entry is always a list
if not isinstance(ret[_id][k], list):
ret[_id][k] = [ret[_id][k]]

# If the current value is not already in the list, append it
if v not in [item['value'] for item in ret[_id][k]]:
ret[_id][k].append(v)
if v_ not in [item['value'] for item in ret[_id][k]]:
ret[_id][k].append(v_)

return ret

Expand Down Expand Up @@ -359,12 +361,10 @@ def _sendQuery(self, query):
:returns: SPARQL query results
"""
LOGGER.debug('Sending SPARQL query')
sparql = SPARQLWrapper(self.sparql_endpoint)
sparql.setQuery(query)
sparql.setReturnFormat(JSON)
self.sparql.setQuery(query)

try:
results = sparql.query().convert()
results = self.sparql.query().convert()
LOGGER.debug('Received SPARQL results')
except Exception as err:
LOGGER.error(f'Error in SPARQL query: {err}')
Expand Down Expand Up @@ -407,9 +407,11 @@ def parse(value: str) -> list:
otherwise the original string.
"""
if '|' in value:
return value.split('|')
LOGGER.debug('Splitting value by "|"')
return value.lstrip('|').rstrip('|').split('|')
elif ', ' in value:
return value.split(', ')
LOGGER.debug('Splitting value by ", "')
return value.lstrip(', ').rstrip(', ').split(', ')
else:
return value

Expand All @@ -424,13 +426,19 @@ def combine_lists(dict_data: dict):
"""
# Extract keys from the dictionary
keys = list(dict_data.keys())
if len(keys) == 1:
LOGGER.debug('Returning un-mondified data')
return dict_data

# Ensure all lists have the same length
length = len(dict_data[keys[0]])
LOGGER.debug(f'Number of keys: {length}')
if not all(len(dict_data[key]) == length for key in keys):
LOGGER.debug('Returning un-mondified data')
return dict_data

# Combine the items into a list of dictionaries
LOGGER.debug(f'Extracting data for: {keys}')
combined_list = [
{key: dict_data[key][i] for key in keys} for i in range(length)
]
Expand Down
4 changes: 2 additions & 2 deletions tests/test_sitemap_process.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def test_sitemap_generator(body):
assert len(sitemap) == 5

common = sitemap.pop('common.xml')
assert len(common) == 2402
assert len(common) == 3134

root = xml.etree.ElementTree.fromstring(common)
assert all(i.tag == j.tag for (i, j) in zip(root, root.findall('url')))
Expand All @@ -79,7 +79,7 @@ def test_sitemap_no_features(body):
assert len(sitemap) == 1

common = sitemap.pop('common.xml')
assert len(common) == 2402
assert len(common) == 3134


def test_sitemap_zip(body):
Expand Down
4 changes: 1 addition & 3 deletions tests/test_sparql_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,9 @@ def test_query(config):
assert base_fields['uri']['type'] == 'string'

fields = p.get_fields()
assert len(fields) == 6
assert len(fields) == 3
for field in base_fields:
assert field in fields
assert fields['country']['type'] == 'string'
assert fields['leader']['type'] == 'string'

results = p.query()
assert len(results['features']) == 8
Expand Down

0 comments on commit 1eb0430

Please sign in to comment.