Merge pull request #13 from UDST/dev

Dev
UDST · Jun 15, 2018 · 155110a · 155110a
2 parents a748346 + e41da4f
commit 155110a
Show file tree

Hide file tree

Showing 17 changed files with 239 additions and 50 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,4 @@
+[run]
+omit =
+    osmnet/**/tests/*
+    */__init__.py
diff --git a/.gitignore b/.gitignore
@@ -10,6 +10,7 @@
 /data
 /configs
 /docs/build
+/scripts
 
 # Test cache
 .cache

diff --git a/.travis.yml b/.travis.yml
@@ -18,12 +18,13 @@ install:
   conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pip numpy=1.10 pandas pytest pyyaml
 - source activate test-environment
 - conda list
-- pip install geopandas Shapely pycodestyle
+- pip install geopandas Shapely pycodestyle coveralls pytest-cov
 - pip install .
 
 script:
 - pycodestyle osmnet
-- py.test
+- py.test --cov osmnet --cov-report term-missing
 
 after_success:
+- coveralls
 - bin/build_docs.sh
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -0,0 +1,12 @@
+#### If you have found an error:
+
+  - check the error message and [documentation](https://udst.github.io/osmnet/index.html)
+  - search the previously opened and closed issues to see if the problem has already been reported
+  - if the problem is with a dependency of OSMnet, please open an issue on the dependency's repo
+  - if the problem is with OSMnet and you think you may have a fix, please submit a PR, otherwise please open an issue in the [issue tracker](https://github.com/UDST/osmnet/issues) following the issue template
+
+#### Making a feature proposal or contributing code:
+
+  - post your requested feature on the [issue tracker](https://github.com/UDST/osmnet/issues) and mark it with a `New feature` label so it can be reviewed
+  - fork the repo, make your change (your code should attempt to conform to OSMnet's existing coding, commenting, and docstring styles), add new or update [unit tests](https://github.com/UDST/osmnet/tree/master/osmnet/tests), and submit a PR
+  - respond to the code review
diff --git a/HISTORY.rst b/HISTORY.rst
@@ -0,0 +1,48 @@
+v0.1.5
+======
+
+2018/6/15
+
+* adds custom_osm_filter parameter
+* update default keep_osm_tags list to include more tags including area
+* add new exceptions to catch for queries that return bad/no data
+* add coveralls support
+
+v0.1.4
+======
+
+2017/4/6
+
+* Better exception handling of KeyError exceptions.
+* Removed unnecessary logs.
+
+v0.1.3
+======
+
+2017/4/6
+
+* Documentation is now generated and upload to the gh-pages branch after each commit on master.
+
+v0.1.2
+======
+
+2017/3/31
+
+* Now version numbers are the same in all the source code.
+
+v0.1.1
+======
+
+2017/3/31
+
+* README file changed from MarkDown to RST format.
+
+v0.1.0
+======
+
+2017/3/31
+
+* Python3 support. Now, Travis run tests over Python2.7 and Python3.5.
+* Travis runs pycodestyle on each commit.
+* Code now conforms pep8 and pycodestyle.
+
diff --git a/README.rst b/README.rst
@@ -1,7 +1,7 @@
 OSMnet
 ======
 
-|Build Status| |Appveyor Build Status|
+|Build Status|  |Coverage Status|  |Appveyor Build Status|
 
 Tools for the extraction of OpenStreetMap (OSM) street network data.
 Intended to be used in tandem with Pandana and UrbanAccess libraries to
@@ -90,4 +90,7 @@ Related UDST libraries
    :target: https://travis-ci.org/UDST/osmnet
 
 .. |Appveyor Build Status| image:: https://ci.appveyor.com/api/projects/status/acuoygyy3l0lqnpv/branch/master?svg=true
-   :target: https://ci.appveyor.com/project/pksohn/osmnet
+   :target: https://ci.appveyor.com/project/pksohn/osmnet
+
+.. |Coverage Status| image:: https://coveralls.io/repos/github/UDST/osmnet/badge.svg?branch=master
+   :target: https://coveralls.io/github/UDST/osmnet?branch=master
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -30,8 +30,8 @@
 project = u'OSMnet'
 copyright = u'2017, UrbanSim Inc.'
 author = u'UrbanSim Inc.'
-version = u'0.1.4'
-release = u'0.1.4'
+version = u'0.1.5'
+release = u'0.1.5'
 language = None
 
 nitpicky = True
@@ -47,6 +47,7 @@
 # -- Options for HTML output ----------------------------------------------
 html_theme = 'sphinx_rtd_theme'
 html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+html_show_sourcelink = False
 # html_theme_options = {}
 
 # paths that contain custom static files (such as style sheets)

diff --git a/docs/source/installation.rst b/docs/source/installation.rst
@@ -15,7 +15,7 @@ Dependencies
 Note for Windows Users when Installing Dependencies
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-If you are a Windows user and you find when importing osmnet you see an error like this: ``ImportError: DLL load failed: The specified module could not be found.`` Most likely one of osmnet's dependencies did not install or compile correctly on your Windows machine. ``geopandas`` requires the dependency package ``fiona`` which requires the dependency package ``gdal``. Windows users should not install these dependencies via conda or pip, instead you should download and install these packages via `Christoph Gohlke Windows python wheels`_: `GDAL Windows Wheel`_ and `Fiona Windows Wheel`_. Download the package that matched your python version and Windows system architecture, then cd into the download directory and install each package using: ``pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl`` and
+If you are a Windows user and you find when importing osmnet you see an error like this: ``ImportError: DLL load failed: The specified module could not be found.`` Most likely one of osmnet's dependencies did not install or compile correctly on your Windows machine. ``geopandas`` requires the dependency package ``fiona`` which requires the dependency package ``gdal``. Windows users should not install these dependencies via conda or pip, instead you should download and install these packages via `Christoph Gohlke Windows python wheels`_: `GDAL Windows Wheel`_ and `Fiona Windows Wheel`_. Download the package that matches your Python version and Windows system architecture, then cd into the download directory and install each package for example using: ``pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl`` and
 ``pip install GDAL-2.1.3-cp27-cp27m-win_amd64.whl``
 If you have already installed these packaged via conda or pip, force a reinstall: ``pip install Fiona-1.7.6-cp27-cp27m-win_amd64.whl --upgrade --force-reinstall`` and
 ``pip install GDAL-2.1.3-cp27-cp27m-win_amd64.whl --upgrade --force-reinstall``
@@ -46,13 +46,13 @@ Development Installation
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 To install use the ``develop`` command rather than ``install``. Make sure you
-are using the latest version of the code base by using git’s ``git pull``
+are using the latest version of the codebase by using git’s ``git pull``
 inside the cloned repository.
 
 To install OSMnet follow these steps:
 
 1. Git clone the `OSMnet repo`_
-2. in the cloned directory run: ``python setup.py develop``
+2. in the cloned directory run: ``python setup.py develop`` or without dependencies: ``python setup.py develop --no-deps``
 
 To update to the latest version:
 

diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst
@@ -10,12 +10,18 @@ Reporting bugs
 ~~~~~~~~~~~~~~~~~~~~~~~~
 Please report any bugs you encounter via `GitHub Issues <https://github.com/UDST/osmnet/issues>`__.
 
+1. Check the error message and documentation
+2. Search the previously opened and closed issues to see if the problem has already been reported
+3. If the problem is with a dependency of OSMnet, please open an issue on the dependency's repo
+4. If the problem is with OSMnet and you think you may have a fix, please submit a PR, otherwise please open an issue in the `issue tracker <https://github.com/UDST/osmnet/issues>`__ following the issue template
+
 Contributing to OSMnet
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 If you have improvements or new features you would like to see in OSMnet:
 
-1. Open a feature request via `GitHub Issues <https://github.com/UDST/osmnet/issues>`__.
-2. Contribute your code from a fork or branch by using a Pull Request and request a review so it can be considered as an addition to the codebase.
+1. Open a feature request via `GitHub Issues <https://github.com/UDST/osmnet/issues>`__ and mark it with a `New feature` label so it can be reviewed
+2. Contribute your code from a fork or branch by using a Pull Request (PR) and request a review so it can be considered as an addition to the codebase. Your code should attempt to conform to OSMnet's existing coding, commenting, and docstring styles. Add new or update the existing `unit tests <https://github.com/UDST/osmnet/tree/master/osmnet/tests>`__
+3. Respond to the code review
 
 License
 ~~~~~~~~

diff --git a/docs/source/osmnet.rst b/docs/source/osmnet.rst
@@ -4,7 +4,7 @@ Using OSMnet
 Creating a graph network from a OSM street network
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Create a ``drive`` (e.g. automobile) or ``walk`` (e.g. pedestrian) graph network comprised of nodes and edges from OpenStreetMap (OSM) street network data via the OverpassAPI. Edges will be weighted by default by distance in units of meters. The resulting graph network is intended to be used with `Pandana`_ network accessibility queries and `UrbanAccess`_ to create an integrated transit and street graph network.
+Create a ``drive`` (e.g. automobile) or ``walk`` (e.g. pedestrian) graph network comprised of nodes and edges from OpenStreetMap (OSM) street network data via the OverpassAPI. Alternatively, custom highway way queries formatted for OverpassAPI can be written and passed to the ``custom_osm_filter`` parameter. Edges will be weighted by default by distance in units of meters. The resulting graph network is intended to be used with `Pandana`_ network accessibility queries and `UrbanAccess`_ to create an integrated transit and street graph network.
 
 .. autofunction:: osmnet.load.network_from_bbox
 

diff --git a/osmnet/config.py b/osmnet/config.py
@@ -60,7 +60,8 @@ def __init__(self,
                  log_filename='osmnet',
                  keep_osm_tags=['name', 'ref', 'highway', 'service', 'bridge',
                                 'tunnel', 'access', 'oneway', 'toll', 'lanes',
-                                'maxspeed', 'hgv', 'hov']):
+                                'maxspeed', 'hgv', 'hov', 'area', 'width',
+                                'est_width', 'junction']):
 
         self.logs_folder = logs_folder
         self.log_file = log_file

diff --git a/osmnet/load.py b/osmnet/load.py
@@ -1,4 +1,4 @@
-# The following functions to download osm data, setup an recursive api request
+# The following functions to download osm data, setup a recursive api request
 # and subdivide bbox queries into smaller bboxes were modified from the
 # osmnx library and used with permission from the author Geoff Boeing
 # osm_net_download, overpass_request, get_pause_duration,
@@ -69,7 +69,8 @@ def osm_filter(network_type):
 
 def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None,
                      network_type='walk', timeout=180, memory=None,
-                     max_query_area_size=50*1000*50*1000):
+                     max_query_area_size=50*1000*50*1000,
+                     custom_osm_filter=None):
     """
     Download OSM ways and nodes within a bounding box from the Overpass API.
 
@@ -97,6 +98,11 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None,
         in: any polygon bigger will get divided up for multiple queries to
         Overpass API (default is 50,000 * 50,000 units (ie, 50km x 50km in
         area, if units are meters))
+    custom_osm_filter : string, optional
+        specify custom arguments for the way["highway"] query to OSM. Must
+        follow Overpass API schema. For
+        example to request highway ways that are service roads use:
+        '["highway"="service"]'
 
     Returns
     -------
@@ -105,7 +111,11 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None,
 
     # create a filter to exclude certain kinds of ways based on the requested
     # network_type
-    request_filter = osm_filter(network_type)
+    if custom_osm_filter is None:
+        request_filter = osm_filter(network_type)
+    else:
+        request_filter = custom_osm_filter
+
     response_jsons_list = []
     response_jsons = []
 
@@ -170,17 +180,22 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None,
     start_time = time.time()
     record_count = len(response_jsons)
 
-    response_jsons_df = pd.DataFrame.from_records(response_jsons, index='id')
-    nodes = response_jsons_df[response_jsons_df['type'] == 'node']
-    nodes = nodes[~nodes.index.duplicated(keep='first')]
-    ways = response_jsons_df[response_jsons_df['type'] == 'way']
-    ways = ways[~ways.index.duplicated(keep='first')]
-    response_jsons_df = pd.concat([nodes, ways], axis=0)
-    response_jsons_df.reset_index(inplace=True)
-    response_jsons = response_jsons_df.to_dict(orient='records')
-    if record_count-len(response_jsons) > 0:
-        log('{:,} duplicate records removed. Took {:,.2f} seconds'.format(
-            record_count-len(response_jsons), time.time()-start_time))
+    if record_count == 0:
+        raise Exception('Query resulted in no data. Check your query '
+                        'parameters: {}'.format(query_str))
+    else:
+        response_jsons_df = pd.DataFrame.from_records(response_jsons,
+                                                      index='id')
+        nodes = response_jsons_df[response_jsons_df['type'] == 'node']
+        nodes = nodes[~nodes.index.duplicated(keep='first')]
+        ways = response_jsons_df[response_jsons_df['type'] == 'way']
+        ways = ways[~ways.index.duplicated(keep='first')]
+        response_jsons_df = pd.concat([nodes, ways], axis=0)
+        response_jsons_df.reset_index(inplace=True)
+        response_jsons = response_jsons_df.to_dict(orient='records')
+        if record_count - len(response_jsons) > 0:
+            log('{:,} duplicate records removed. Took {:,.2f} seconds'.format(
+                record_count - len(response_jsons), time.time() - start_time))
 
     return {'elements': response_jsons}
 
@@ -595,7 +610,8 @@ def parse_network_osm_query(data):
 
 def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type,
                  timeout=180, memory=None,
-                 max_query_area_size=50*1000*50*1000):
+                 max_query_area_size=50*1000*50*1000,
+                 custom_osm_filter=None):
     """
     Get DataFrames of OSM data in a bounding box.
 
@@ -623,6 +639,11 @@ def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type,
         in: any polygon bigger will get divided up for multiple queries to
         Overpass API (default is 50,000 * 50,000 units (ie, 50km x 50km in
         area, if units are meters))
+    custom_osm_filter : string, optional
+        specify custom arguments for the way["highway"] query to OSM. Must
+        follow Overpass API schema. For
+        example to request highway ways that are service roads use:
+        '["highway"="service"]'
 
     Returns
     -------
@@ -633,7 +654,8 @@ def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type,
         osm_net_download(lat_max=lat_max, lat_min=lat_min, lng_min=lng_min,
                          lng_max=lng_max, network_type=network_type,
                          timeout=timeout, memory=memory,
-                         max_query_area_size=max_query_area_size))
+                         max_query_area_size=max_query_area_size,
+                         custom_osm_filter=custom_osm_filter))
 
 
 def intersection_nodes(waynodes):
@@ -730,18 +752,23 @@ def pairwise(l):
                     pairs.append(col_dict)
 
     pairs = pd.DataFrame.from_records(pairs)
-    pairs.index = pd.MultiIndex.from_arrays([pairs['from_id'].values,
-                                             pairs['to_id'].values])
-    log('Edge node pairs completed. Took {:,.2f} seconds'
-        .format(time.time()-start_time))
+    if pairs.empty:
+        raise Exception('Query resulted in no connected node pairs. Check '
+                        'your query parameters or bounding box')
+    else:
+        pairs.index = pd.MultiIndex.from_arrays([pairs['from_id'].values,
+                                                 pairs['to_id'].values])
+        log('Edge node pairs completed. Took {:,.2f} seconds'
+            .format(time.time()-start_time))
 
-    return pairs
+        return pairs
 
 
 def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None,
                       bbox=None, network_type='walk', two_way=True,
                       timeout=180, memory=None,
-                      max_query_area_size=50*1000*50*1000):
+                      max_query_area_size=50*1000*50*1000,
+                      custom_osm_filter=None):
     """
     Make a graph network from a bounding lat/lon box composed of nodes and
     edges for use in Pandana street network accessibility calculations.
@@ -773,7 +800,8 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None,
     network_type : {'walk', 'drive'}, optional
         Specify the network type where value of 'walk' includes roadways where
         pedestrians are allowed and pedestrian pathways and 'drive' includes
-        driveable roadways. Default is walk.
+        driveable roadways. To use a custom definition see the
+        custom_osm_filter parameter. Default is walk.
     two_way : bool, optional
         Whether the routes are two-way. If True, node pairs will only
         occur once.
@@ -787,10 +815,11 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None,
         in: any polygon bigger will get divided up for multiple queries to
         Overpass API (default is 50,000 * 50,000 units (ie, 50km x 50km in
         area, if units are meters))
-    remove_lcn : bool, optional
-        remove low connectivity nodes from the resulting pandana network.
-        This ensures the resulting network does not have nodes that are
-        unconnected from the rest of the larger network
+    custom_osm_filter : string, optional
+        specify custom arguments for the way["highway"] query to OSM. Must
+        follow Overpass API schema. For
+        example to request highway ways that are service roads use:
+        '["highway"="service"]'
 
     Returns
     -------
@@ -821,7 +850,8 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None,
     nodes, ways, waynodes = ways_in_bbox(
         lat_min=lat_min, lng_min=lng_min, lat_max=lat_max, lng_max=lng_max,
         network_type=network_type, timeout=timeout,
-        memory=memory, max_query_area_size=max_query_area_size)
+        memory=memory, max_query_area_size=max_query_area_size,
+        custom_osm_filter=custom_osm_filter)
     log('Returning OSM data with {:,} nodes and {:,} ways...'
         .format(len(nodes), len(ways)))
 

diff --git a/osmnet/tests/test_config.py b/osmnet/tests/test_config.py
@@ -0,0 +1,23 @@
+import pytest
+
+import osmnet.config as config
+
+
+@pytest.fixture(scope='module')
+def default_config():
+    # Default config settings
+    return {'log_file': True,
+            'log_name': 'osmnet',
+            'log_filename': 'osmnet',
+            'logs_folder': 'logs',
+            'keep_osm_tags': ['name', 'ref', 'highway', 'service', 'bridge',
+                              'tunnel', 'access', 'oneway', 'toll', 'lanes',
+                              'maxspeed', 'hgv', 'hov', 'area', 'width',
+                              'est_width', 'junction'],
+            'log_console': False}
+
+
+def test_config_defaults(default_config):
+    settings = config.osmnet_config()
+    config.format_check(settings.to_dict())
+    assert settings.to_dict() == default_config