From d55fb7c3d3a717e115863e5903c4332b1bc705ba Mon Sep 17 00:00:00 2001 From: lmnoel Date: Tue, 15 May 2018 13:13:15 -0500 Subject: [PATCH 01/17] Added argument for custom OSM filter --- osmnet/load.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/osmnet/load.py b/osmnet/load.py index c3b50b7..09a6f91 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -69,7 +69,8 @@ def osm_filter(network_type): def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, network_type='walk', timeout=180, memory=None, - max_query_area_size=50*1000*50*1000): + max_query_area_size=50*1000*50*1000, + custom_osm_filter=None): """ Download OSM ways and nodes within a bounding box from the Overpass API. @@ -97,6 +98,8 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, in: any polygon bigger will get divided up for multiple queries to Overpass API (default is 50,000 * 50,000 units (ie, 50km x 50km in area, if units are meters)) + custom_osm_filter : string, optional + specify custom arguments for the query to OSM Returns ------- @@ -105,7 +108,11 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, # create a filter to exclude certain kinds of ways based on the requested # network_type - request_filter = osm_filter(network_type) + if custom_osm_filter is None: + request_filter = osm_filter(network_type) + else: + request_filter = custom_osm_filter + response_jsons_list = [] response_jsons = [] @@ -595,7 +602,8 @@ def parse_network_osm_query(data): def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, timeout=180, memory=None, - max_query_area_size=50*1000*50*1000): + max_query_area_size=50*1000*50*1000, + custom_osm_filter=None): """ Get DataFrames of OSM data in a bounding box. @@ -623,6 +631,8 @@ def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, in: any polygon bigger will get divided up for multiple queries to Overpass API (default is 50,000 * 50,000 units (ie, 50km x 50km in area, if units are meters)) + custom_osm_filter : string, optional + specify custom arguments for the query to OSM Returns ------- @@ -633,7 +643,8 @@ def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, osm_net_download(lat_max=lat_max, lat_min=lat_min, lng_min=lng_min, lng_max=lng_max, network_type=network_type, timeout=timeout, memory=memory, - max_query_area_size=max_query_area_size)) + max_query_area_size=max_query_area_size, + custom_osm_filter=custom_osm_filter)) def intersection_nodes(waynodes): @@ -741,7 +752,8 @@ def pairwise(l): def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, bbox=None, network_type='walk', two_way=True, timeout=180, memory=None, - max_query_area_size=50*1000*50*1000): + max_query_area_size=50*1000*50*1000, + custom_osm_filter=None): """ Make a graph network from a bounding lat/lon box composed of nodes and edges for use in Pandana street network accessibility calculations. @@ -791,6 +803,8 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, remove low connectivity nodes from the resulting pandana network. This ensures the resulting network does not have nodes that are unconnected from the rest of the larger network + custom_osm_filter : string, optional + specify custom arguments for the query to OSM Returns ------- @@ -821,7 +835,8 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, nodes, ways, waynodes = ways_in_bbox( lat_min=lat_min, lng_min=lng_min, lat_max=lat_max, lng_max=lng_max, network_type=network_type, timeout=timeout, - memory=memory, max_query_area_size=max_query_area_size) + memory=memory, max_query_area_size=max_query_area_size, + custom_osm_filter=custom_osm_filter) log('Returning OSM data with {:,} nodes and {:,} ways...' .format(len(nodes), len(ways))) From cb32ad2abb1f8297351f2ba04625c2a09a6d6d5a Mon Sep 17 00:00:00 2001 From: sablanchard Date: Sun, 10 Jun 2018 18:11:22 -0700 Subject: [PATCH 02/17] add exception when query returns no data --- osmnet/load.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/osmnet/load.py b/osmnet/load.py index 09a6f91..e496dfe 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -177,17 +177,21 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, start_time = time.time() record_count = len(response_jsons) - response_jsons_df = pd.DataFrame.from_records(response_jsons, index='id') - nodes = response_jsons_df[response_jsons_df['type'] == 'node'] - nodes = nodes[~nodes.index.duplicated(keep='first')] - ways = response_jsons_df[response_jsons_df['type'] == 'way'] - ways = ways[~ways.index.duplicated(keep='first')] - response_jsons_df = pd.concat([nodes, ways], axis=0) - response_jsons_df.reset_index(inplace=True) - response_jsons = response_jsons_df.to_dict(orient='records') - if record_count-len(response_jsons) > 0: - log('{:,} duplicate records removed. Took {:,.2f} seconds'.format( - record_count-len(response_jsons), time.time()-start_time)) + if record_count == 0: + raise Exception('Query resulted in no data. Check your query ' + 'parameters: {}'.format(query_str)) + else: + response_jsons_df = pd.DataFrame.from_records(response_jsons, index='id') + nodes = response_jsons_df[response_jsons_df['type'] == 'node'] + nodes = nodes[~nodes.index.duplicated(keep='first')] + ways = response_jsons_df[response_jsons_df['type'] == 'way'] + ways = ways[~ways.index.duplicated(keep='first')] + response_jsons_df = pd.concat([nodes, ways], axis=0) + response_jsons_df.reset_index(inplace=True) + response_jsons = response_jsons_df.to_dict(orient='records') + if record_count-len(response_jsons) > 0: + log('{:,} duplicate records removed. Took {:,.2f} seconds'.format( + record_count-len(response_jsons), time.time()-start_time)) return {'elements': response_jsons} From ce6d0a6793fd405f82989e57778a297815b3ef07 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Sun, 10 Jun 2018 18:20:47 -0700 Subject: [PATCH 03/17] update docstrings --- osmnet/load.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/osmnet/load.py b/osmnet/load.py index e496dfe..17d906a 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -99,7 +99,10 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, Overpass API (default is 50,000 * 50,000 units (ie, 50km x 50km in area, if units are meters)) custom_osm_filter : string, optional - specify custom arguments for the query to OSM + specify custom arguments for the way["highway"] query to OSM. Must + follow overpass api schema. For + example to request highway ways that are service roads use: + '["highway"="service"]' Returns ------- @@ -636,7 +639,10 @@ def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, Overpass API (default is 50,000 * 50,000 units (ie, 50km x 50km in area, if units are meters)) custom_osm_filter : string, optional - specify custom arguments for the query to OSM + specify custom arguments for the way["highway"] query to OSM. Must + follow overpass api schema. For + example to request highway ways that are service roads use: + '["highway"="service"]' Returns ------- @@ -789,7 +795,8 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, network_type : {'walk', 'drive'}, optional Specify the network type where value of 'walk' includes roadways where pedestrians are allowed and pedestrian pathways and 'drive' includes - driveable roadways. Default is walk. + driveable roadways. To use a custom definition see the + custom_osm_filter parameter. Default is walk. two_way : bool, optional Whether the routes are two-way. If True, node pairs will only occur once. @@ -803,12 +810,11 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, in: any polygon bigger will get divided up for multiple queries to Overpass API (default is 50,000 * 50,000 units (ie, 50km x 50km in area, if units are meters)) - remove_lcn : bool, optional - remove low connectivity nodes from the resulting pandana network. - This ensures the resulting network does not have nodes that are - unconnected from the rest of the larger network custom_osm_filter : string, optional - specify custom arguments for the query to OSM + specify custom arguments for the way["highway"] query to OSM. Must + follow overpass api schema. For + example to request highway ways that are service roads use: + '["highway"="service"]' Returns ------- From 23f3a259465099a313b807e8fa055a389c21263e Mon Sep 17 00:00:00 2001 From: sablanchard Date: Sun, 10 Jun 2018 21:39:43 -0700 Subject: [PATCH 04/17] add exception when no pairs are found, minor pep8 formatting --- osmnet/load.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/osmnet/load.py b/osmnet/load.py index 17d906a..0f6875f 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -184,7 +184,8 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, raise Exception('Query resulted in no data. Check your query ' 'parameters: {}'.format(query_str)) else: - response_jsons_df = pd.DataFrame.from_records(response_jsons, index='id') + response_jsons_df = pd.DataFrame.from_records(response_jsons, + index='id') nodes = response_jsons_df[response_jsons_df['type'] == 'node'] nodes = nodes[~nodes.index.duplicated(keep='first')] ways = response_jsons_df[response_jsons_df['type'] == 'way'] @@ -192,9 +193,9 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, response_jsons_df = pd.concat([nodes, ways], axis=0) response_jsons_df.reset_index(inplace=True) response_jsons = response_jsons_df.to_dict(orient='records') - if record_count-len(response_jsons) > 0: + if record_count - len(response_jsons) > 0: log('{:,} duplicate records removed. Took {:,.2f} seconds'.format( - record_count-len(response_jsons), time.time()-start_time)) + record_count - len(response_jsons), time.time() - start_time)) return {'elements': response_jsons} @@ -751,12 +752,16 @@ def pairwise(l): pairs.append(col_dict) pairs = pd.DataFrame.from_records(pairs) - pairs.index = pd.MultiIndex.from_arrays([pairs['from_id'].values, - pairs['to_id'].values]) - log('Edge node pairs completed. Took {:,.2f} seconds' - .format(time.time()-start_time)) + if pairs.empty: + raise Exception('Query resulted in no connected node pairs. Check ' + 'your query parameters or bounding box') + else: + pairs.index = pd.MultiIndex.from_arrays([pairs['from_id'].values, + pairs['to_id'].values]) + log('Edge node pairs completed. Took {:,.2f} seconds' + .format(time.time()-start_time)) - return pairs + return pairs def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, From 9e1e1e1ff2d08b8578d1f13071861f68c6067aee Mon Sep 17 00:00:00 2001 From: sablanchard Date: Sun, 10 Jun 2018 21:40:24 -0700 Subject: [PATCH 05/17] update default config keep_osm_tags list --- osmnet/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/osmnet/config.py b/osmnet/config.py index 20101a9..8f60ebc 100644 --- a/osmnet/config.py +++ b/osmnet/config.py @@ -60,7 +60,8 @@ def __init__(self, log_filename='osmnet', keep_osm_tags=['name', 'ref', 'highway', 'service', 'bridge', 'tunnel', 'access', 'oneway', 'toll', 'lanes', - 'maxspeed', 'hgv', 'hov']): + 'maxspeed', 'hgv', 'hov', 'area', 'width', + 'est_width', 'junction']): self.logs_folder = logs_folder self.log_file = log_file From 810f349c640b7705dc228c82f0aeac6618ce9773 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Sun, 10 Jun 2018 21:41:05 -0700 Subject: [PATCH 06/17] added coverage --- .coveragerc | 4 ++++ .gitignore | 1 + .travis.yml | 5 +++-- README.rst | 7 +++++-- 4 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..d2e2dfb --- /dev/null +++ b/.coveragerc @@ -0,0 +1,4 @@ +[run] +omit = + osmnet/tests/* + */__init__.py diff --git a/.gitignore b/.gitignore index 3d88fc8..c47a11d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ /data /configs /docs/build +/scripts # Test cache .cache diff --git a/.travis.yml b/.travis.yml index d4bd42c..16fca3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,12 +18,13 @@ install: conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pip numpy=1.10 pandas pytest pyyaml - source activate test-environment - conda list -- pip install geopandas Shapely pycodestyle +- pip install geopandas Shapely pycodestyle coveralls pytest-cov - pip install . script: - pycodestyle osmnet -- py.test +- py.test --cov osmnet --cov-report term-missing after_success: +- coveralls - bin/build_docs.sh diff --git a/README.rst b/README.rst index a850206..64634e8 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ OSMnet ====== -|Build Status| |Appveyor Build Status| +|Build Status| |Coverage Status| |Appveyor Build Status| Tools for the extraction of OpenStreetMap (OSM) street network data. Intended to be used in tandem with Pandana and UrbanAccess libraries to @@ -90,4 +90,7 @@ Related UDST libraries :target: https://travis-ci.org/UDST/osmnet .. |Appveyor Build Status| image:: https://ci.appveyor.com/api/projects/status/acuoygyy3l0lqnpv/branch/master?svg=true - :target: https://ci.appveyor.com/project/pksohn/osmnet \ No newline at end of file + :target: https://ci.appveyor.com/project/pksohn/osmnet + +.. |Coverage Status| image:: https://coveralls.io/repos/github/UDST/osmnet/badge.svg?branch=master + :target: https://coveralls.io/github/UDST/osmnet?branch=master \ No newline at end of file From 9ea86ccdadd23d68247a93d7370ebe6e888aa7aa Mon Sep 17 00:00:00 2001 From: sablanchard Date: Sun, 10 Jun 2018 21:41:28 -0700 Subject: [PATCH 07/17] updated and added new unit tests --- osmnet/tests/test_config.py | 23 ++++++++++++++ osmnet/tests/test_load.py | 62 +++++++++++++++++++++++++++++++++---- osmnet/tests/test_utils.py | 11 ++++++- 3 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 osmnet/tests/test_config.py diff --git a/osmnet/tests/test_config.py b/osmnet/tests/test_config.py new file mode 100644 index 0000000..77bd846 --- /dev/null +++ b/osmnet/tests/test_config.py @@ -0,0 +1,23 @@ +import pytest + +import osmnet.config as config + + +@pytest.fixture(scope='module') +def default_config(): + # Default config settings + return {'log_file': True, + 'log_name': 'osmnet', + 'log_filename': 'osmnet', + 'logs_folder': 'logs', + 'keep_osm_tags': ['name', 'ref', 'highway', 'service', 'bridge', + 'tunnel', 'access', 'oneway', 'toll', 'lanes', + 'maxspeed', 'hgv', 'hov', 'area', 'width', + 'est_width', 'junction'], + 'log_console': False} + + +def test_config_defaults(default_config): + settings = config.osmnet_config() + config.format_check(settings.to_dict()) + assert settings.to_dict() == default_config diff --git a/osmnet/tests/test_load.py b/osmnet/tests/test_load.py index 562a8b3..cf0c1d1 100644 --- a/osmnet/tests/test_load.py +++ b/osmnet/tests/test_load.py @@ -1,6 +1,7 @@ import numpy.testing as npt import pandas.util.testing as pdt import pytest +import shapely.geometry as geometry import osmnet.load as load @@ -34,6 +35,18 @@ def bbox4(): -122.2701716423, 37.8241329692) +@pytest.fixture +def bbox5(): + return (-122.2965574674, 37.8038112007, + -122.2935963086, 37.8056400922) + + +@pytest.fixture +def simple_polygon(): + polygon = geometry.Polygon([[0, 0], [1, 0], [1, 1], [0, 1]]) + return polygon + + @pytest.fixture(scope='module') def query_data1(bbox1): lat_min, lng_max, lat_max, lng_min = bbox1 @@ -72,11 +85,11 @@ def dataframes2(query_data2): def test_make_osm_query(query_data1): assert isinstance(query_data1, dict) - assert len(query_data1['elements']) == 42 + assert len(query_data1['elements']) == 26 assert len([e for e in query_data1['elements'] - if e['type'] == 'node']) == 39 + if e['type'] == 'node']) == 22 assert len([e for e in query_data1['elements'] - if e['type'] == 'way']) == 3 + if e['type'] == 'way']) == 4 def test_process_node(): @@ -142,9 +155,9 @@ def test_process_way(): def test_parse_network_osm_query(dataframes1): nodes, ways, waynodes = dataframes1 - assert len(nodes) == 39 - assert len(ways) == 3 - assert len(waynodes.index.unique()) == 3 + assert len(nodes) == 22 + assert len(ways) == 4 + assert len(waynodes.index.unique()) == 4 def test_parse_network_osm_query_raises(): @@ -160,6 +173,35 @@ def test_parse_network_osm_query_raises(): load.parse_network_osm_query(data) +def test_overpass_request_raises(bbox5): + lat_min, lng_max, lat_max, lng_min = bbox5 + query_template = '[out:json][timeout:{timeout}]{maxsize};(way["highway"]' \ + '{filters}({lat_min:.8f},{lng_max:.8f},{lat_max:.8f},' \ + '{lng_min:.8f});>;);out;' + query_str = query_template.format(lat_max=lat_max, lat_min=lat_min, + lng_min=lng_min, lng_max=lng_max, + filters=load.osm_filter('walk'), + timeout=0, maxsize='') + with pytest.raises(Exception): + load.overpass_request(data={'data': query_str}) + + +def test_get_pause_duration(): + error_pause_duration = load.get_pause_duration(recursive_delay=5, + default_duration=10) + print error_pause_duration == 0 + + +def test_quadrat_cut_geometry(simple_polygon): + multipolygon = load.quadrat_cut_geometry(geometry=simple_polygon, + quadrat_width=0.5, + min_num=3, + buffer_amount=1e-9) + + assert isinstance(multipolygon, geometry.MultiPolygon) + assert len(multipolygon) == 4 + + def test_ways_in_bbox(bbox1, dataframes1): lat_min, lng_max, lat_max, lng_min = bbox1 nodes, ways, waynodes = load.ways_in_bbox(lat_min=lat_min, lng_min=lng_min, @@ -246,3 +288,11 @@ def test_column_names(bbox4): col_list = ['distance', 'from', 'to'] for col in col_list: assert col in edges.columns + + +def test_custom_query_pass(bbox5): + nodes, edges = load.network_from_bbox( + bbox=bbox5, custom_osm_filter='["highway"="service"]' + ) + assert len(nodes) == 22 + assert len(edges) == 30 diff --git a/osmnet/tests/test_utils.py b/osmnet/tests/test_utils.py index 8498eac..60948b2 100644 --- a/osmnet/tests/test_utils.py +++ b/osmnet/tests/test_utils.py @@ -1,6 +1,7 @@ import numpy.testing as npt +import logging as lg -from osmnet.utils import great_circle_dist as gcd +from osmnet.utils import great_circle_dist as gcd, log def test_gcd(): @@ -14,3 +15,11 @@ def test_gcd(): expected = 864456.76162966 npt.assert_allclose(gcd(lat1, lon1, lat2, lon2), expected) + + +def test_logging(): + + log('test debug error message', level=lg.DEBUG) + log('test info error message', level=lg.INFO) + log('test warning error message', level=lg.WARNING) + log('test error message', level=lg.ERROR) From 429dbb317f7f04e56f38d79a5427229aeef79744 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Sun, 10 Jun 2018 21:44:47 -0700 Subject: [PATCH 08/17] bumped up version and added HISTORY --- HISTORY.rst | 38 ++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 HISTORY.rst diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..cde10df --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,38 @@ +v0.1.4 +====== + +2017/4/6 + +* Better exception handling of KeyError exceptions. +* Removed unnecessary logs. + +v0.1.3 +====== + +2017/4/6 + +* Documentation is now generated and upload to the gh-pages branch after each commit on master. + +v0.1.2 +====== + +2017/3/31 + +* Now version numbers are the same in all the source code. + +v0.1.1 +====== + +2017/3/31 + +* README file changed from MarkDown to RST format. + +v0.1.0 +====== + +2017/3/31 + +* Python3 support. Now, Travis run tests over Python2.7 and Python3.5. +* Travis runs pycodestyle on each commit. +* Code now conforms pep8 and pycodestyle. + diff --git a/setup.py b/setup.py index cb523b4..d94c794 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ setup( name='osmnet', - version='0.1.4', + version='0.1.5', license='AGPL', description=('Tools for the extraction of OpenStreetMap street network ' 'data for use in Pandana accessibility analyses.'), From d5097c590147362f7952cb91aac03ecdd78babbc Mon Sep 17 00:00:00 2001 From: sablanchard Date: Sun, 10 Jun 2018 21:57:28 -0700 Subject: [PATCH 09/17] fixed test --- osmnet/tests/test_load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmnet/tests/test_load.py b/osmnet/tests/test_load.py index cf0c1d1..bbfc89f 100644 --- a/osmnet/tests/test_load.py +++ b/osmnet/tests/test_load.py @@ -189,7 +189,7 @@ def test_overpass_request_raises(bbox5): def test_get_pause_duration(): error_pause_duration = load.get_pause_duration(recursive_delay=5, default_duration=10) - print error_pause_duration == 0 + assert error_pause_duration == 0 def test_quadrat_cut_geometry(simple_polygon): From 1d0fb03ac00c2d55b1d893b40e3fd4e5d9e64152 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 11 Jun 2018 13:50:27 -0700 Subject: [PATCH 10/17] updated documentation added contribution guidelines --- CONTRIBUTING.md | 12 ++++++++++++ docs/source/conf.py | 5 +++-- docs/source/installation.rst | 6 +++--- docs/source/introduction.rst | 10 ++++++++-- docs/source/osmnet.rst | 2 +- osmnet/load.py | 8 ++++---- osmnet/tests/test_load.py | 1 + 7 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5a58914 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,12 @@ +#### If you have found an error: + + - check the error message and [documentation](https://udst.github.io/osmnet/index.html) + - search the previously opened and closed issues to see if the problem has already been reported + - if the problem is with a dependency of OSMnet, please open an issue on the dependency's repo + - if the problem is with OSMnet and you think you may have a fix, please submit a PR, otherwise please open an issue in the [issue tracker](https://github.com/UDST/osmnet/issues) following the issue template + +#### Making a feature proposal or contributing code: + + - post your requested feature on the [issue tracker](https://github.com/UDST/osmnet/issues) and mark it with a `New feature` label so it can be reviewed + - fork the repo, make your change (your code should attempt to conform to OSMnet's existing coding, commenting, and docstring styles), add new or update [unit tests](https://github.com/UDST/osmnet/tree/master/osmnet/tests), and submit a PR + - respond to the code review diff --git a/docs/source/conf.py b/docs/source/conf.py index a6d283b..7a98924 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,8 +30,8 @@ project = u'OSMnet' copyright = u'2017, UrbanSim Inc.' author = u'UrbanSim Inc.' -version = u'0.1.4' -release = u'0.1.4' +version = u'0.1.5' +release = u'0.1.5' language = None nitpicky = True @@ -47,6 +47,7 @@ # -- Options for HTML output ---------------------------------------------- html_theme = 'sphinx_rtd_theme' html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +html_show_sourcelink = False # html_theme_options = {} # paths that contain custom static files (such as style sheets) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 93bb634..29f43d0 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -15,7 +15,7 @@ Dependencies Note for Windows Users when Installing Dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you are a Windows user and you find when importing osmnet you see an error like this: ``ImportError: DLL load failed: The specified module could not be found.`` Most likely one of osmnet's dependencies did not install or compile correctly on your Windows machine. ``geopandas`` requires the dependency package ``fiona`` which requires the dependency package ``gdal``. Windows users should not install these dependencies via conda or pip, instead you should download and install these packages via `Christoph Gohlke Windows python wheels`_: `GDAL Windows Wheel`_ and `Fiona Windows Wheel`_. Download the package that matched your python version and Windows system architecture, then cd into the download directory and install each package using: ``pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl`` and +If you are a Windows user and you find when importing osmnet you see an error like this: ``ImportError: DLL load failed: The specified module could not be found.`` Most likely one of osmnet's dependencies did not install or compile correctly on your Windows machine. ``geopandas`` requires the dependency package ``fiona`` which requires the dependency package ``gdal``. Windows users should not install these dependencies via conda or pip, instead you should download and install these packages via `Christoph Gohlke Windows python wheels`_: `GDAL Windows Wheel`_ and `Fiona Windows Wheel`_. Download the package that matches your Python version and Windows system architecture, then cd into the download directory and install each package for example using: ``pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl`` and ``pip install GDAL-2.1.3-cp27-cp27m-win_amd64.whl`` If you have already installed these packaged via conda or pip, force a reinstall: ``pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl --upgrade --force-reinstall`` and ``pip install GDAL-2.1.3-cp27-cp27m-win_amd64.whl --upgrade --force-reinstall`` @@ -46,13 +46,13 @@ Development Installation ^^^^^^^^^^^^^^^^^^^^^^^^^^ To install use the ``develop`` command rather than ``install``. Make sure you -are using the latest version of the code base by using git’s ``git pull`` +are using the latest version of the codebase by using git’s ``git pull`` inside the cloned repository. To install OSMnet follow these steps: 1. Git clone the `OSMnet repo`_ -2. in the cloned directory run: ``python setup.py develop`` +2. in the cloned directory run: ``python setup.py develop`` or without dependencies: ``python setup.py develop --no-deps`` To update to the latest version: diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index 1dc7c10..0e1ad84 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -10,12 +10,18 @@ Reporting bugs ~~~~~~~~~~~~~~~~~~~~~~~~ Please report any bugs you encounter via `GitHub Issues `__. +1. Check the error message and documentation +2. Search the previously opened and closed issues to see if the problem has already been reported +3. If the problem is with a dependency of OSMnet, please open an issue on the dependency's repo +4. If the problem is with OSMnet and you think you may have a fix, please submit a PR, otherwise please open an issue in the `issue tracker `__ following the issue template + Contributing to OSMnet ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you have improvements or new features you would like to see in OSMnet: -1. Open a feature request via `GitHub Issues `__. -2. Contribute your code from a fork or branch by using a Pull Request and request a review so it can be considered as an addition to the codebase. +1. Open a feature request via `GitHub Issues `__ and mark it with a `New feature` label so it can be reviewed +2. Contribute your code from a fork or branch by using a Pull Request (PR) and request a review so it can be considered as an addition to the codebase. Your code should attempt to conform to OSMnet's existing coding, commenting, and docstring styles. Add new or update the existing `unit tests `__ +3. Respond to the code review License ~~~~~~~~ diff --git a/docs/source/osmnet.rst b/docs/source/osmnet.rst index 3b16ec4..5ae7885 100644 --- a/docs/source/osmnet.rst +++ b/docs/source/osmnet.rst @@ -4,7 +4,7 @@ Using OSMnet Creating a graph network from a OSM street network ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Create a ``drive`` (e.g. automobile) or ``walk`` (e.g. pedestrian) graph network comprised of nodes and edges from OpenStreetMap (OSM) street network data via the OverpassAPI. Edges will be weighted by default by distance in units of meters. The resulting graph network is intended to be used with `Pandana`_ network accessibility queries and `UrbanAccess`_ to create an integrated transit and street graph network. +Create a ``drive`` (e.g. automobile) or ``walk`` (e.g. pedestrian) graph network comprised of nodes and edges from OpenStreetMap (OSM) street network data via the OverpassAPI. Alternatively, custom highway way queries formatted for OverpassAPI can be written and passed to the ``custom_osm_filter`` parameter. Edges will be weighted by default by distance in units of meters. The resulting graph network is intended to be used with `Pandana`_ network accessibility queries and `UrbanAccess`_ to create an integrated transit and street graph network. .. autofunction:: osmnet.load.network_from_bbox diff --git a/osmnet/load.py b/osmnet/load.py index 0f6875f..70d4533 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -1,4 +1,4 @@ -# The following functions to download osm data, setup an recursive api request +# The following functions to download osm data, setup a recursive api request # and subdivide bbox queries into smaller bboxes were modified from the # osmnx library and used with permission from the author Geoff Boeing # osm_net_download, overpass_request, get_pause_duration, @@ -100,7 +100,7 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, area, if units are meters)) custom_osm_filter : string, optional specify custom arguments for the way["highway"] query to OSM. Must - follow overpass api schema. For + follow Overpass API schema. For example to request highway ways that are service roads use: '["highway"="service"]' @@ -641,7 +641,7 @@ def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, area, if units are meters)) custom_osm_filter : string, optional specify custom arguments for the way["highway"] query to OSM. Must - follow overpass api schema. For + follow Overpass API schema. For example to request highway ways that are service roads use: '["highway"="service"]' @@ -817,7 +817,7 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, area, if units are meters)) custom_osm_filter : string, optional specify custom arguments for the way["highway"] query to OSM. Must - follow overpass api schema. For + follow Overpass API schema. For example to request highway ways that are service roads use: '["highway"="service"]' diff --git a/osmnet/tests/test_load.py b/osmnet/tests/test_load.py index bbfc89f..ebd8b1d 100644 --- a/osmnet/tests/test_load.py +++ b/osmnet/tests/test_load.py @@ -296,3 +296,4 @@ def test_custom_query_pass(bbox5): ) assert len(nodes) == 22 assert len(edges) == 30 + assert edges['highway'].unique() == 'service' \ No newline at end of file From 31a8aeff516d35b126bc1e54857572f8baec7995 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 11 Jun 2018 13:57:54 -0700 Subject: [PATCH 11/17] pep8 fix --- osmnet/tests/test_load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmnet/tests/test_load.py b/osmnet/tests/test_load.py index ebd8b1d..e3aee06 100644 --- a/osmnet/tests/test_load.py +++ b/osmnet/tests/test_load.py @@ -296,4 +296,4 @@ def test_custom_query_pass(bbox5): ) assert len(nodes) == 22 assert len(edges) == 30 - assert edges['highway'].unique() == 'service' \ No newline at end of file + assert edges['highway'].unique() == 'service' From 8e2a3d4b2734be039a990709a6a9c7042ad0cbca Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 11 Jun 2018 14:22:41 -0700 Subject: [PATCH 12/17] coverage adjustment --- .coveragerc | 2 +- osmnet/tests/test_utils.py | 10 ++++------ osmnet/utils.py | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.coveragerc b/.coveragerc index d2e2dfb..8d32801 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,4 @@ [run] omit = - osmnet/tests/* + osmnet/**/tests/* */__init__.py diff --git a/osmnet/tests/test_utils.py b/osmnet/tests/test_utils.py index 60948b2..aa55f4c 100644 --- a/osmnet/tests/test_utils.py +++ b/osmnet/tests/test_utils.py @@ -3,7 +3,6 @@ from osmnet.utils import great_circle_dist as gcd, log - def test_gcd(): # tested against geopy # https://geopy.readthedocs.org/en/latest/#module-geopy.distance @@ -18,8 +17,7 @@ def test_gcd(): def test_logging(): - - log('test debug error message', level=lg.DEBUG) - log('test info error message', level=lg.INFO) - log('test warning error message', level=lg.WARNING) - log('test error message', level=lg.ERROR) + log('test debug message', level=lg.DEBUG) + log('test info message', level=lg.INFO) + log('test warning message', level=lg.WARNING) + log('test error message', level=lg.ERROR) \ No newline at end of file diff --git a/osmnet/utils.py b/osmnet/utils.py index 0913d2c..c113234 100644 --- a/osmnet/utils.py +++ b/osmnet/utils.py @@ -76,7 +76,7 @@ def log(message, level=None, name=None, filename=None): if filename is None: filename = config.settings.log_filename - if config.settings.log_file: + if config.settings.log_file: # pragma: no cover # get the current logger or create a new one then log message at # requested level logger = get_logger(level=level, name=name, filename=filename) From 51426b3e51392513182c45cafd6c8fbbb295a756 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 11 Jun 2018 14:24:07 -0700 Subject: [PATCH 13/17] coverage fix --- osmnet/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osmnet/utils.py b/osmnet/utils.py index c113234..e9496a7 100644 --- a/osmnet/utils.py +++ b/osmnet/utils.py @@ -76,7 +76,7 @@ def log(message, level=None, name=None, filename=None): if filename is None: filename = config.settings.log_filename - if config.settings.log_file: # pragma: no cover + if config.settings.log_file: # get the current logger or create a new one then log message at # requested level logger = get_logger(level=level, name=name, filename=filename) @@ -91,7 +91,7 @@ def log(message, level=None, name=None, filename=None): # if logging to console is turned on, convert message to ascii and print # to the console only - if config.settings.log_console: + if config.settings.log_console: # pragma: no cover # capture current stdout, then switch it to the console, print the # message, then switch back to what had been the stdout # this prevents logging to notebook - instead, it goes to console From 77e12f5b13437371720b2170c3c59a7d1d14be14 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 11 Jun 2018 14:33:49 -0700 Subject: [PATCH 14/17] fixed pep8 --- osmnet/tests/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/osmnet/tests/test_utils.py b/osmnet/tests/test_utils.py index aa55f4c..cb26bab 100644 --- a/osmnet/tests/test_utils.py +++ b/osmnet/tests/test_utils.py @@ -3,6 +3,7 @@ from osmnet.utils import great_circle_dist as gcd, log + def test_gcd(): # tested against geopy # https://geopy.readthedocs.org/en/latest/#module-geopy.distance @@ -20,4 +21,4 @@ def test_logging(): log('test debug message', level=lg.DEBUG) log('test info message', level=lg.INFO) log('test warning message', level=lg.WARNING) - log('test error message', level=lg.ERROR) \ No newline at end of file + log('test error message', level=lg.ERROR) From af69114a0d001600f5ce467ec32cfe8dbc160a6a Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 11 Jun 2018 14:49:55 -0700 Subject: [PATCH 15/17] pep8 fix --- osmnet/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmnet/utils.py b/osmnet/utils.py index e9496a7..54887e5 100644 --- a/osmnet/utils.py +++ b/osmnet/utils.py @@ -91,7 +91,7 @@ def log(message, level=None, name=None, filename=None): # if logging to console is turned on, convert message to ascii and print # to the console only - if config.settings.log_console: # pragma: no cover + if config.settings.log_console: # pragma: no cover # capture current stdout, then switch it to the console, print the # message, then switch back to what had been the stdout # this prevents logging to notebook - instead, it goes to console From f6c850f5a0c7f1a6c12014c7f89c686d25dcad00 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Fri, 15 Jun 2018 11:09:14 -0700 Subject: [PATCH 16/17] update history --- HISTORY.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index cde10df..d5959cc 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,3 +1,13 @@ +v0.1.5 +====== + +2018/6/15 + +* adds custom_osm_filter parameter +* update default keep_osm_tags list to include more tags including area +* add new exceptions to catch for queries that return bad/no data +* add coveralls support + v0.1.4 ====== From e41da4f27995d9c7af0ec41350b455060bc8b82b Mon Sep 17 00:00:00 2001 From: sablanchard Date: Fri, 15 Jun 2018 11:23:42 -0700 Subject: [PATCH 17/17] fixed travis --- osmnet/tests/test_load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmnet/tests/test_load.py b/osmnet/tests/test_load.py index e3aee06..c07e5d6 100644 --- a/osmnet/tests/test_load.py +++ b/osmnet/tests/test_load.py @@ -189,7 +189,7 @@ def test_overpass_request_raises(bbox5): def test_get_pause_duration(): error_pause_duration = load.get_pause_duration(recursive_delay=5, default_duration=10) - assert error_pause_duration == 0 + assert error_pause_duration >= 0 def test_quadrat_cut_geometry(simple_polygon):