diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..8d32801 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,4 @@ +[run] +omit = + osmnet/**/tests/* + */__init__.py diff --git a/.gitignore b/.gitignore index 3d88fc8..c47a11d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ /data /configs /docs/build +/scripts # Test cache .cache diff --git a/.travis.yml b/.travis.yml index d4bd42c..16fca3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,12 +18,13 @@ install: conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pip numpy=1.10 pandas pytest pyyaml - source activate test-environment - conda list -- pip install geopandas Shapely pycodestyle +- pip install geopandas Shapely pycodestyle coveralls pytest-cov - pip install . script: - pycodestyle osmnet -- py.test +- py.test --cov osmnet --cov-report term-missing after_success: +- coveralls - bin/build_docs.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5a58914 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,12 @@ +#### If you have found an error: + + - check the error message and [documentation](https://udst.github.io/osmnet/index.html) + - search the previously opened and closed issues to see if the problem has already been reported + - if the problem is with a dependency of OSMnet, please open an issue on the dependency's repo + - if the problem is with OSMnet and you think you may have a fix, please submit a PR, otherwise please open an issue in the [issue tracker](https://github.com/UDST/osmnet/issues) following the issue template + +#### Making a feature proposal or contributing code: + + - post your requested feature on the [issue tracker](https://github.com/UDST/osmnet/issues) and mark it with a `New feature` label so it can be reviewed + - fork the repo, make your change (your code should attempt to conform to OSMnet's existing coding, commenting, and docstring styles), add new or update [unit tests](https://github.com/UDST/osmnet/tree/master/osmnet/tests), and submit a PR + - respond to the code review diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..d5959cc --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,48 @@ +v0.1.5 +====== + +2018/6/15 + +* adds custom_osm_filter parameter +* update default keep_osm_tags list to include more tags including area +* add new exceptions to catch for queries that return bad/no data +* add coveralls support + +v0.1.4 +====== + +2017/4/6 + +* Better exception handling of KeyError exceptions. +* Removed unnecessary logs. + +v0.1.3 +====== + +2017/4/6 + +* Documentation is now generated and upload to the gh-pages branch after each commit on master. + +v0.1.2 +====== + +2017/3/31 + +* Now version numbers are the same in all the source code. + +v0.1.1 +====== + +2017/3/31 + +* README file changed from MarkDown to RST format. + +v0.1.0 +====== + +2017/3/31 + +* Python3 support. Now, Travis run tests over Python2.7 and Python3.5. +* Travis runs pycodestyle on each commit. +* Code now conforms pep8 and pycodestyle. + diff --git a/README.rst b/README.rst index a850206..64634e8 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ OSMnet ====== -|Build Status| |Appveyor Build Status| +|Build Status| |Coverage Status| |Appveyor Build Status| Tools for the extraction of OpenStreetMap (OSM) street network data. Intended to be used in tandem with Pandana and UrbanAccess libraries to @@ -90,4 +90,7 @@ Related UDST libraries :target: https://travis-ci.org/UDST/osmnet .. |Appveyor Build Status| image:: https://ci.appveyor.com/api/projects/status/acuoygyy3l0lqnpv/branch/master?svg=true - :target: https://ci.appveyor.com/project/pksohn/osmnet \ No newline at end of file + :target: https://ci.appveyor.com/project/pksohn/osmnet + +.. |Coverage Status| image:: https://coveralls.io/repos/github/UDST/osmnet/badge.svg?branch=master + :target: https://coveralls.io/github/UDST/osmnet?branch=master \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index a6d283b..7a98924 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,8 +30,8 @@ project = u'OSMnet' copyright = u'2017, UrbanSim Inc.' author = u'UrbanSim Inc.' -version = u'0.1.4' -release = u'0.1.4' +version = u'0.1.5' +release = u'0.1.5' language = None nitpicky = True @@ -47,6 +47,7 @@ # -- Options for HTML output ---------------------------------------------- html_theme = 'sphinx_rtd_theme' html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +html_show_sourcelink = False # html_theme_options = {} # paths that contain custom static files (such as style sheets) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 93bb634..29f43d0 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -15,7 +15,7 @@ Dependencies Note for Windows Users when Installing Dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you are a Windows user and you find when importing osmnet you see an error like this: ``ImportError: DLL load failed: The specified module could not be found.`` Most likely one of osmnet's dependencies did not install or compile correctly on your Windows machine. ``geopandas`` requires the dependency package ``fiona`` which requires the dependency package ``gdal``. Windows users should not install these dependencies via conda or pip, instead you should download and install these packages via `Christoph Gohlke Windows python wheels`_: `GDAL Windows Wheel`_ and `Fiona Windows Wheel`_. Download the package that matched your python version and Windows system architecture, then cd into the download directory and install each package using: ``pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl`` and +If you are a Windows user and you find when importing osmnet you see an error like this: ``ImportError: DLL load failed: The specified module could not be found.`` Most likely one of osmnet's dependencies did not install or compile correctly on your Windows machine. ``geopandas`` requires the dependency package ``fiona`` which requires the dependency package ``gdal``. Windows users should not install these dependencies via conda or pip, instead you should download and install these packages via `Christoph Gohlke Windows python wheels`_: `GDAL Windows Wheel`_ and `Fiona Windows Wheel`_. Download the package that matches your Python version and Windows system architecture, then cd into the download directory and install each package for example using: ``pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl`` and ``pip install GDAL-2.1.3-cp27-cp27m-win_amd64.whl`` If you have already installed these packaged via conda or pip, force a reinstall: ``pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl --upgrade --force-reinstall`` and ``pip install GDAL-2.1.3-cp27-cp27m-win_amd64.whl --upgrade --force-reinstall`` @@ -46,13 +46,13 @@ Development Installation ^^^^^^^^^^^^^^^^^^^^^^^^^^ To install use the ``develop`` command rather than ``install``. Make sure you -are using the latest version of the code base by using git’s ``git pull`` +are using the latest version of the codebase by using git’s ``git pull`` inside the cloned repository. To install OSMnet follow these steps: 1. Git clone the `OSMnet repo`_ -2. in the cloned directory run: ``python setup.py develop`` +2. in the cloned directory run: ``python setup.py develop`` or without dependencies: ``python setup.py develop --no-deps`` To update to the latest version: diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index 1dc7c10..0e1ad84 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -10,12 +10,18 @@ Reporting bugs ~~~~~~~~~~~~~~~~~~~~~~~~ Please report any bugs you encounter via `GitHub Issues `__. +1. Check the error message and documentation +2. Search the previously opened and closed issues to see if the problem has already been reported +3. If the problem is with a dependency of OSMnet, please open an issue on the dependency's repo +4. If the problem is with OSMnet and you think you may have a fix, please submit a PR, otherwise please open an issue in the `issue tracker `__ following the issue template + Contributing to OSMnet ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you have improvements or new features you would like to see in OSMnet: -1. Open a feature request via `GitHub Issues `__. -2. Contribute your code from a fork or branch by using a Pull Request and request a review so it can be considered as an addition to the codebase. +1. Open a feature request via `GitHub Issues `__ and mark it with a `New feature` label so it can be reviewed +2. Contribute your code from a fork or branch by using a Pull Request (PR) and request a review so it can be considered as an addition to the codebase. Your code should attempt to conform to OSMnet's existing coding, commenting, and docstring styles. Add new or update the existing `unit tests `__ +3. Respond to the code review License ~~~~~~~~ diff --git a/docs/source/osmnet.rst b/docs/source/osmnet.rst index 3b16ec4..5ae7885 100644 --- a/docs/source/osmnet.rst +++ b/docs/source/osmnet.rst @@ -4,7 +4,7 @@ Using OSMnet Creating a graph network from a OSM street network ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Create a ``drive`` (e.g. automobile) or ``walk`` (e.g. pedestrian) graph network comprised of nodes and edges from OpenStreetMap (OSM) street network data via the OverpassAPI. Edges will be weighted by default by distance in units of meters. The resulting graph network is intended to be used with `Pandana`_ network accessibility queries and `UrbanAccess`_ to create an integrated transit and street graph network. +Create a ``drive`` (e.g. automobile) or ``walk`` (e.g. pedestrian) graph network comprised of nodes and edges from OpenStreetMap (OSM) street network data via the OverpassAPI. Alternatively, custom highway way queries formatted for OverpassAPI can be written and passed to the ``custom_osm_filter`` parameter. Edges will be weighted by default by distance in units of meters. The resulting graph network is intended to be used with `Pandana`_ network accessibility queries and `UrbanAccess`_ to create an integrated transit and street graph network. .. autofunction:: osmnet.load.network_from_bbox diff --git a/osmnet/config.py b/osmnet/config.py index 20101a9..8f60ebc 100644 --- a/osmnet/config.py +++ b/osmnet/config.py @@ -60,7 +60,8 @@ def __init__(self, log_filename='osmnet', keep_osm_tags=['name', 'ref', 'highway', 'service', 'bridge', 'tunnel', 'access', 'oneway', 'toll', 'lanes', - 'maxspeed', 'hgv', 'hov']): + 'maxspeed', 'hgv', 'hov', 'area', 'width', + 'est_width', 'junction']): self.logs_folder = logs_folder self.log_file = log_file diff --git a/osmnet/load.py b/osmnet/load.py index c3b50b7..70d4533 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -1,4 +1,4 @@ -# The following functions to download osm data, setup an recursive api request +# The following functions to download osm data, setup a recursive api request # and subdivide bbox queries into smaller bboxes were modified from the # osmnx library and used with permission from the author Geoff Boeing # osm_net_download, overpass_request, get_pause_duration, @@ -69,7 +69,8 @@ def osm_filter(network_type): def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, network_type='walk', timeout=180, memory=None, - max_query_area_size=50*1000*50*1000): + max_query_area_size=50*1000*50*1000, + custom_osm_filter=None): """ Download OSM ways and nodes within a bounding box from the Overpass API. @@ -97,6 +98,11 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, in: any polygon bigger will get divided up for multiple queries to Overpass API (default is 50,000 * 50,000 units (ie, 50km x 50km in area, if units are meters)) + custom_osm_filter : string, optional + specify custom arguments for the way["highway"] query to OSM. Must + follow Overpass API schema. For + example to request highway ways that are service roads use: + '["highway"="service"]' Returns ------- @@ -105,7 +111,11 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, # create a filter to exclude certain kinds of ways based on the requested # network_type - request_filter = osm_filter(network_type) + if custom_osm_filter is None: + request_filter = osm_filter(network_type) + else: + request_filter = custom_osm_filter + response_jsons_list = [] response_jsons = [] @@ -170,17 +180,22 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, start_time = time.time() record_count = len(response_jsons) - response_jsons_df = pd.DataFrame.from_records(response_jsons, index='id') - nodes = response_jsons_df[response_jsons_df['type'] == 'node'] - nodes = nodes[~nodes.index.duplicated(keep='first')] - ways = response_jsons_df[response_jsons_df['type'] == 'way'] - ways = ways[~ways.index.duplicated(keep='first')] - response_jsons_df = pd.concat([nodes, ways], axis=0) - response_jsons_df.reset_index(inplace=True) - response_jsons = response_jsons_df.to_dict(orient='records') - if record_count-len(response_jsons) > 0: - log('{:,} duplicate records removed. Took {:,.2f} seconds'.format( - record_count-len(response_jsons), time.time()-start_time)) + if record_count == 0: + raise Exception('Query resulted in no data. Check your query ' + 'parameters: {}'.format(query_str)) + else: + response_jsons_df = pd.DataFrame.from_records(response_jsons, + index='id') + nodes = response_jsons_df[response_jsons_df['type'] == 'node'] + nodes = nodes[~nodes.index.duplicated(keep='first')] + ways = response_jsons_df[response_jsons_df['type'] == 'way'] + ways = ways[~ways.index.duplicated(keep='first')] + response_jsons_df = pd.concat([nodes, ways], axis=0) + response_jsons_df.reset_index(inplace=True) + response_jsons = response_jsons_df.to_dict(orient='records') + if record_count - len(response_jsons) > 0: + log('{:,} duplicate records removed. Took {:,.2f} seconds'.format( + record_count - len(response_jsons), time.time() - start_time)) return {'elements': response_jsons} @@ -595,7 +610,8 @@ def parse_network_osm_query(data): def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, timeout=180, memory=None, - max_query_area_size=50*1000*50*1000): + max_query_area_size=50*1000*50*1000, + custom_osm_filter=None): """ Get DataFrames of OSM data in a bounding box. @@ -623,6 +639,11 @@ def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, in: any polygon bigger will get divided up for multiple queries to Overpass API (default is 50,000 * 50,000 units (ie, 50km x 50km in area, if units are meters)) + custom_osm_filter : string, optional + specify custom arguments for the way["highway"] query to OSM. Must + follow Overpass API schema. For + example to request highway ways that are service roads use: + '["highway"="service"]' Returns ------- @@ -633,7 +654,8 @@ def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, osm_net_download(lat_max=lat_max, lat_min=lat_min, lng_min=lng_min, lng_max=lng_max, network_type=network_type, timeout=timeout, memory=memory, - max_query_area_size=max_query_area_size)) + max_query_area_size=max_query_area_size, + custom_osm_filter=custom_osm_filter)) def intersection_nodes(waynodes): @@ -730,18 +752,23 @@ def pairwise(l): pairs.append(col_dict) pairs = pd.DataFrame.from_records(pairs) - pairs.index = pd.MultiIndex.from_arrays([pairs['from_id'].values, - pairs['to_id'].values]) - log('Edge node pairs completed. Took {:,.2f} seconds' - .format(time.time()-start_time)) + if pairs.empty: + raise Exception('Query resulted in no connected node pairs. Check ' + 'your query parameters or bounding box') + else: + pairs.index = pd.MultiIndex.from_arrays([pairs['from_id'].values, + pairs['to_id'].values]) + log('Edge node pairs completed. Took {:,.2f} seconds' + .format(time.time()-start_time)) - return pairs + return pairs def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, bbox=None, network_type='walk', two_way=True, timeout=180, memory=None, - max_query_area_size=50*1000*50*1000): + max_query_area_size=50*1000*50*1000, + custom_osm_filter=None): """ Make a graph network from a bounding lat/lon box composed of nodes and edges for use in Pandana street network accessibility calculations. @@ -773,7 +800,8 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, network_type : {'walk', 'drive'}, optional Specify the network type where value of 'walk' includes roadways where pedestrians are allowed and pedestrian pathways and 'drive' includes - driveable roadways. Default is walk. + driveable roadways. To use a custom definition see the + custom_osm_filter parameter. Default is walk. two_way : bool, optional Whether the routes are two-way. If True, node pairs will only occur once. @@ -787,10 +815,11 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, in: any polygon bigger will get divided up for multiple queries to Overpass API (default is 50,000 * 50,000 units (ie, 50km x 50km in area, if units are meters)) - remove_lcn : bool, optional - remove low connectivity nodes from the resulting pandana network. - This ensures the resulting network does not have nodes that are - unconnected from the rest of the larger network + custom_osm_filter : string, optional + specify custom arguments for the way["highway"] query to OSM. Must + follow Overpass API schema. For + example to request highway ways that are service roads use: + '["highway"="service"]' Returns ------- @@ -821,7 +850,8 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, nodes, ways, waynodes = ways_in_bbox( lat_min=lat_min, lng_min=lng_min, lat_max=lat_max, lng_max=lng_max, network_type=network_type, timeout=timeout, - memory=memory, max_query_area_size=max_query_area_size) + memory=memory, max_query_area_size=max_query_area_size, + custom_osm_filter=custom_osm_filter) log('Returning OSM data with {:,} nodes and {:,} ways...' .format(len(nodes), len(ways))) diff --git a/osmnet/tests/test_config.py b/osmnet/tests/test_config.py new file mode 100644 index 0000000..77bd846 --- /dev/null +++ b/osmnet/tests/test_config.py @@ -0,0 +1,23 @@ +import pytest + +import osmnet.config as config + + +@pytest.fixture(scope='module') +def default_config(): + # Default config settings + return {'log_file': True, + 'log_name': 'osmnet', + 'log_filename': 'osmnet', + 'logs_folder': 'logs', + 'keep_osm_tags': ['name', 'ref', 'highway', 'service', 'bridge', + 'tunnel', 'access', 'oneway', 'toll', 'lanes', + 'maxspeed', 'hgv', 'hov', 'area', 'width', + 'est_width', 'junction'], + 'log_console': False} + + +def test_config_defaults(default_config): + settings = config.osmnet_config() + config.format_check(settings.to_dict()) + assert settings.to_dict() == default_config diff --git a/osmnet/tests/test_load.py b/osmnet/tests/test_load.py index 562a8b3..c07e5d6 100644 --- a/osmnet/tests/test_load.py +++ b/osmnet/tests/test_load.py @@ -1,6 +1,7 @@ import numpy.testing as npt import pandas.util.testing as pdt import pytest +import shapely.geometry as geometry import osmnet.load as load @@ -34,6 +35,18 @@ def bbox4(): -122.2701716423, 37.8241329692) +@pytest.fixture +def bbox5(): + return (-122.2965574674, 37.8038112007, + -122.2935963086, 37.8056400922) + + +@pytest.fixture +def simple_polygon(): + polygon = geometry.Polygon([[0, 0], [1, 0], [1, 1], [0, 1]]) + return polygon + + @pytest.fixture(scope='module') def query_data1(bbox1): lat_min, lng_max, lat_max, lng_min = bbox1 @@ -72,11 +85,11 @@ def dataframes2(query_data2): def test_make_osm_query(query_data1): assert isinstance(query_data1, dict) - assert len(query_data1['elements']) == 42 + assert len(query_data1['elements']) == 26 assert len([e for e in query_data1['elements'] - if e['type'] == 'node']) == 39 + if e['type'] == 'node']) == 22 assert len([e for e in query_data1['elements'] - if e['type'] == 'way']) == 3 + if e['type'] == 'way']) == 4 def test_process_node(): @@ -142,9 +155,9 @@ def test_process_way(): def test_parse_network_osm_query(dataframes1): nodes, ways, waynodes = dataframes1 - assert len(nodes) == 39 - assert len(ways) == 3 - assert len(waynodes.index.unique()) == 3 + assert len(nodes) == 22 + assert len(ways) == 4 + assert len(waynodes.index.unique()) == 4 def test_parse_network_osm_query_raises(): @@ -160,6 +173,35 @@ def test_parse_network_osm_query_raises(): load.parse_network_osm_query(data) +def test_overpass_request_raises(bbox5): + lat_min, lng_max, lat_max, lng_min = bbox5 + query_template = '[out:json][timeout:{timeout}]{maxsize};(way["highway"]' \ + '{filters}({lat_min:.8f},{lng_max:.8f},{lat_max:.8f},' \ + '{lng_min:.8f});>;);out;' + query_str = query_template.format(lat_max=lat_max, lat_min=lat_min, + lng_min=lng_min, lng_max=lng_max, + filters=load.osm_filter('walk'), + timeout=0, maxsize='') + with pytest.raises(Exception): + load.overpass_request(data={'data': query_str}) + + +def test_get_pause_duration(): + error_pause_duration = load.get_pause_duration(recursive_delay=5, + default_duration=10) + assert error_pause_duration >= 0 + + +def test_quadrat_cut_geometry(simple_polygon): + multipolygon = load.quadrat_cut_geometry(geometry=simple_polygon, + quadrat_width=0.5, + min_num=3, + buffer_amount=1e-9) + + assert isinstance(multipolygon, geometry.MultiPolygon) + assert len(multipolygon) == 4 + + def test_ways_in_bbox(bbox1, dataframes1): lat_min, lng_max, lat_max, lng_min = bbox1 nodes, ways, waynodes = load.ways_in_bbox(lat_min=lat_min, lng_min=lng_min, @@ -246,3 +288,12 @@ def test_column_names(bbox4): col_list = ['distance', 'from', 'to'] for col in col_list: assert col in edges.columns + + +def test_custom_query_pass(bbox5): + nodes, edges = load.network_from_bbox( + bbox=bbox5, custom_osm_filter='["highway"="service"]' + ) + assert len(nodes) == 22 + assert len(edges) == 30 + assert edges['highway'].unique() == 'service' diff --git a/osmnet/tests/test_utils.py b/osmnet/tests/test_utils.py index 8498eac..cb26bab 100644 --- a/osmnet/tests/test_utils.py +++ b/osmnet/tests/test_utils.py @@ -1,6 +1,7 @@ import numpy.testing as npt +import logging as lg -from osmnet.utils import great_circle_dist as gcd +from osmnet.utils import great_circle_dist as gcd, log def test_gcd(): @@ -14,3 +15,10 @@ def test_gcd(): expected = 864456.76162966 npt.assert_allclose(gcd(lat1, lon1, lat2, lon2), expected) + + +def test_logging(): + log('test debug message', level=lg.DEBUG) + log('test info message', level=lg.INFO) + log('test warning message', level=lg.WARNING) + log('test error message', level=lg.ERROR) diff --git a/osmnet/utils.py b/osmnet/utils.py index 0913d2c..54887e5 100644 --- a/osmnet/utils.py +++ b/osmnet/utils.py @@ -91,7 +91,7 @@ def log(message, level=None, name=None, filename=None): # if logging to console is turned on, convert message to ascii and print # to the console only - if config.settings.log_console: + if config.settings.log_console: # pragma: no cover # capture current stdout, then switch it to the console, print the # message, then switch back to what had been the stdout # this prevents logging to notebook - instead, it goes to console diff --git a/setup.py b/setup.py index cb523b4..d94c794 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ setup( name='osmnet', - version='0.1.4', + version='0.1.5', license='AGPL', description=('Tools for the extraction of OpenStreetMap street network ' 'data for use in Pandana accessibility analyses.'),