more flexible time periods, mandatory schedule specs, input and outpu…

…t formats (#295)

* Provide more flexibility for defining mandatory schedule specifications.

* provide more flexibility for time periods.

* Improve consistency in slicing and fix code syntax issues.

* Consistency and code syntax.

* code syntax.

* code syntax fixes.

* syntax fix.

* Add necessary change to test script settings.

* Modify relevant section of the settings.yml for tests and examples.

* restore 'hours' as deprecated for backwards compatibility.

* Add default value for period_minutes for backwards compatibilitiy.

* create develop branch so we can merge features here before merging (releasing) to master

* provide more flexibility for time periods.

* Improve consistency in slicing and fix code syntax issues.

* Consistency and code syntax.

* code syntax fixes.

* Modify relevant section of the settings.yml for tests and examples.

* restore 'hours' as deprecated for backwards compatibility.

* Add default value for period_minutes for backwards compatibilitiy.

* Fix overflows

* Complete testing of the flexible time periods.

* pep8

* test some floats too.

* pep8

* start with csvs and write hdf5 outputs if desired  (#290)

* add h5_store option to write_tables

* keep csv files as the default

* read input tables from CSV

* update docs, drop scripts no longer needed, include example files in package

* formatting

Co-authored-by: Blake <[email protected]>

* Update travis.yml

* Pandas 1.0 fixes

Co-authored-by: Clint Daniels <[email protected]>
Co-authored-by: Blake <[email protected]>

.gitignore

@@ -65,3 +65,5 @@ docs/_build/
 *.cprof
 *.lprof
 
+# macOS
+*.DS_Store

.travis.yml

@@ -2,31 +2,42 @@ language: python
 sudo: false
 python:
   - '2.7'
-  - '3.6'
+  - '3.7'
 install:
-- if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]];
-  then wget http://repo.continuum.io/miniconda/Miniconda-3.7.0-Linux-x86_64.sh -O miniconda.sh;
-  else wget http://repo.continuum.io/miniconda/Miniconda3-3.7.0-Linux-x86_64.sh -O miniconda.sh; fi
+- wget http://repo.continuum.io/miniconda/Miniconda-3.7.0-Linux-x86_64.sh -O miniconda.sh
 - bash miniconda.sh -b -p $HOME/miniconda
 - export PATH="$HOME/miniconda/bin:$PATH"
 - hash -r
 - conda config --set always_yes yes --set changeps1 no
 - conda update -q conda
 - conda info -a
 - |
-  conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION cytoolz numpy pandas pip pytables pyyaml toolz psutil future
+  conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION future
 - source activate test-environment
-- pip install openmatrix zbox
 - conda install pytest pytest-cov coveralls pycodestyle
-- conda install sphinx numpydoc sphinx_rtd_theme
 - pip install .
 - pip freeze
 script:
 - pycodestyle activitysim
 - py.test --cov activitysim --cov-report term-missing
 after_success:
 - coveralls
-- bin/build_docs.sh
+# Build docs
+- pip install sphinx numpydoc sphinx_rtd_theme
+- cd docs
+- make clean
+- make html
+- touch _build/html/.nojekyll
+
+deploy:
+  provider: pages
+  local_dir: docs/_build/html
+  skip_cleanup: true
+  github_token: $GH_TOKEN
+  keep_history: true
+  on:
+    branch: master
+
 notifications:
   slack:
     secure: Dpp+zBrnPGBHXrYWjwHy/bnHvhINfepSIiViwKfBZizBvTDvzSJfu6gCH+/lQ3squF3D4qTWwxB+LQ9V6KTYhuma8vQVisyneI6ARjUI/qgX6aJjuvmDDGPk6DVeDow7+aCLZ8VEHRhSjwy+dv0Ij0rxI6I94xPVwXUkk7ZjcK0=

MANIFEST.in

@@ -2,3 +2,7 @@ include ez_setup.py
 include README.rst
 include activitysim\abm\test\data\mtc_asim.h5
 include activitysim\abm\test\data\skims.omx
+include activitysim\abm\test\data\households.csv
+include activitysim\abm\test\data\persons.csv
+include activitysim\abm\test\data\land_use.csv
+include activitysim\abm\test\data\override_hh_ids.csv

README.md

@@ -15,4 +15,4 @@ and benefit from contributions of other agency partners.
 
 ## Documentation
 
-https://activitysim.github.io/activitysim 
+https://activitysim.github.io/activitysim  

activitysim/abm/models/initialize.py

@@ -23,6 +23,22 @@
 from activitysim.abm.tables import shadow_pricing
 
 
+# We are using the naming conventions in the mtc_asim.h5 example
+# file for our default list. This provides backwards compatibility
+# with previous versions of ActivitySim in which only 'input_store'
+# is given in the settings file.
+DEFAULT_TABLE_LIST = [
+    {'tablename': 'households',
+     'h5_tablename': 'households',
+     'index_col': 'household_id'},
+    {'tablename': 'persons',
+     'h5_tablename': 'persons',
+     'index_col': 'person_id'},
+    {'tablename': 'land_use',
+     'h5_tablename': 'land_use_taz',
+     'index_col': 'TAZ'}
+]
+
 logger = logging.getLogger(__name__)
 
 
@@ -109,6 +125,18 @@ def preload_injectables():
     inject.add_step('write_data_dictionary', write_data_dictionary)
     inject.add_step('write_tables', write_tables)
 
+    table_list = config.setting('input_table_list')
+
+    # default ActivitySim table names and indices
+    if table_list is None:
+        logger.warn(
+            "No 'input_table_list' found in settings. This will be a "
+            "required setting in upcoming versions of ActivitySim.")
+
+        new_settings = inject.get_injectable('settings')
+        new_settings['input_table_list'] = DEFAULT_TABLE_LIST
+        inject.add_injectable('settings', new_settings)
+
     t0 = tracing.print_elapsed_time()
 
     # FIXME - still want to do this?

activitysim/abm/models/mandatory_scheduling.py

@@ -74,13 +74,9 @@ def mandatory_tour_scheduling(tours,
             mandatory_tours.tour_type.where(~is_university_tour, 'univ')
 
     # - spec dict segmented by primary_purpose
-    work_spec = simulate.read_model_spec(file_name='tour_scheduling_work.csv')
-    school_spec = simulate.read_model_spec(file_name='tour_scheduling_school.csv')
-    segment_specs = {
-        'work': work_spec,
-        'school': school_spec,
-        'univ': school_spec
-    }
+    specs = model_settings.get('SPEC', [])
+    segment_specs = {segment: simulate.read_model_spec(file_name=spec)
+                     for segment, spec in specs.items()}
 
     logger.info("Running mandatory_tour_scheduling with %d tours", len(tours))
     tdd_choices, timetable = vts.vectorize_tour_scheduling(

activitysim/abm/models/util/expressions.py

@@ -6,6 +6,7 @@
 
 import os
 import logging
+import warnings
 
 import numpy as np
 import pandas as pd
@@ -164,13 +165,13 @@ def assign_columns(df, model_settings, locals_dict={}, trace_label=None):
 # helpers
 # ##################################################################################################
 
-def skim_time_period_label(time):
+def skim_time_period_label(time_period):
     """
     convert time period times to skim time period labels (e.g. 9 -> 'AM')
 
     Parameters
     ----------
-    time : pandas Series
+    time_period : pandas Series
 
     Returns
     -------
@@ -180,12 +181,35 @@ def skim_time_period_label(time):
 
     skim_time_periods = config.setting('skim_time_periods')
 
+    # Default to 60 minute time periods
+    period_minutes = 60
+    if 'period_minutes' in skim_time_periods.keys():
+        period_minutes = skim_time_periods['period_minutes']
+
+    # Default to a day
+    model_time_window_min = 1440
+    if ('time_window') in skim_time_periods.keys():
+        model_time_window_min = skim_time_periods['time_window']
+
+    # Check to make sure the intervals result in no remainder time throught 24 hour day
+    assert 0 == model_time_window_min % period_minutes
+    total_periods = model_time_window_min / period_minutes
+
     # FIXME - eventually test and use np version always?
-    if np.isscalar(time):
-        bin = np.digitize([time % 24], skim_time_periods['hours'])[0] - 1
+    period_label = 'periods'
+    if 'hours' in skim_time_periods.keys():
+        period_label = 'hours'
+        warnings.warn('`skim_time_periods` key `hours` in settings.yml will be removed in '
+                      'future verions. Use `periods` instead',
+                      FutureWarning)
+
+    if np.isscalar(time_period):
+        bin = np.digitize([time_period % total_periods],
+                          skim_time_periods[period_label], right=True)[0] - 1
         return skim_time_periods['labels'][bin]
 
-    return pd.cut(time, skim_time_periods['hours'], labels=skim_time_periods['labels']).astype(str)
+    return pd.cut(time_period, skim_time_periods[period_label],
+                  labels=skim_time_periods['labels'], right=True).astype(str)
 
 
 def annotate_preprocessors(

activitysim/abm/tables/__init__.py

@@ -3,8 +3,6 @@
 
 from __future__ import absolute_import
 
-from . import input_store
-
 from . import households
 from . import persons
 from . import landuse

activitysim/abm/tables/households.py

@@ -15,7 +15,7 @@
 from activitysim.core import pipeline
 from activitysim.core import inject
 
-from .input_store import read_input_table
+from activitysim.core.input import read_input_table
 
 logger = logging.getLogger(__name__)
 
@@ -86,8 +86,6 @@ def households(households_sample_size, override_hh_ids, trace_hh_id):
 
     logger.info("loaded households %s" % (df.shape,))
 
-    df.index.name = 'household_id'
-
     # FIXME - pathological knowledge of name of chunk_id column used by chunked_choosers_by_chunk_id
     assert 'chunk_id' not in df.columns
     df['chunk_id'] = pd.Series(list(range(len(df))), df.index)

activitysim/abm/tables/landuse.py

@@ -8,20 +8,18 @@
 import logging
 
 from activitysim.core import inject
-from .input_store import read_input_table
+from activitysim.core.input import read_input_table
 
 logger = logging.getLogger(__name__)
 
 
 @inject.table()
 def land_use():
 
-    df = read_input_table("land_use_taz")
+    df = read_input_table("land_use")
 
     logger.info("loaded land_use %s" % (df.shape,))
 
-    df.index.name = 'TAZ'
-
     # replace table function with dataframe
     inject.add_table('land_use', df)
 

activitysim/abm/tables/persons.py

@@ -11,7 +11,7 @@
 from activitysim.core import inject
 from activitysim.core import tracing
 
-from .input_store import read_input_table
+from activitysim.core.input import read_input_table
 
 logger = logging.getLogger(__name__)
 
@@ -34,8 +34,6 @@ def persons(households, trace_hh_id):
 
     logger.info("loaded persons %s" % (df.shape,))
 
-    df.index.name = 'person_id'
-
     # replace table function with dataframe
     inject.add_table('persons', df)
 

activitysim/abm/tables/skims.py

@@ -15,6 +15,8 @@
 import multiprocessing
 
 from collections import OrderedDict
+from functools import reduce
+from operator import mul
 
 import numpy as np
 import openmatrix as omx
@@ -90,7 +92,7 @@ def get_skim_info(omx_file_path, tags_to_load=None):
 
     if MAX_BLOCK_BYTES:
         max_block_items = MAX_BLOCK_BYTES // np.dtype(skim_dtype).itemsize
-        max_skims_per_block = max_block_items // np.prod(omx_shape)
+        max_skims_per_block = max_block_items // multiply_large_numbers(omx_shape)
     else:
         max_skims_per_block = num_skims
 
@@ -149,14 +151,14 @@ def block_name(block):
 def buffers_for_skims(skim_info, shared=False):
 
     skim_dtype = skim_info['dtype']
-    omx_shape = [np.float64(x) for x in skim_info['omx_shape']]
+    omx_shape = skim_info['omx_shape']
     blocks = skim_info['blocks']
 
     skim_buffers = {}
     for block_name, block_size in iteritems(blocks):
 
         # buffer_size must be int (or p2.7 long), not np.int64
-        buffer_size = int(np.prod(omx_shape) * block_size)
+        buffer_size = int(multiply_large_numbers(omx_shape) * block_size)
 
         csz = buffer_size * np.dtype(skim_dtype).itemsize
         logger.info("allocating shared buffer %s for %s (%s) matrices (%s)" %
@@ -191,7 +193,7 @@ def skim_data_from_buffers(skim_buffers, skim_info):
     for block_name, block_size in iteritems(blocks):
         skims_shape = omx_shape + (block_size,)
         block_buffer = skim_buffers[block_name]
-        assert len(block_buffer) == int(np.prod(skims_shape))
+        assert len(block_buffer) == int(multiply_large_numbers(skims_shape))
         block_data = np.frombuffer(block_buffer, dtype=skim_dtype).reshape(skims_shape)
         skim_data.append(block_data)
 
@@ -261,6 +263,10 @@ def skim_dict(data_dir, settings):
     return skim_dict
 
 
+def multiply_large_numbers(list_of_numbers):
+    return reduce(mul, list_of_numbers)
+
+
 @inject.injectable(cache=True)
 def skim_stack(skim_dict):
 

activitysim/abm/test/configs/mandatory_tour_scheduling.yaml

@@ -21,6 +21,10 @@ LOGSUM_SETTINGS: tour_mode_choice.yaml
 
 SEGMENT_COL: primary_purpose
 
+SPEC:
+  work: tour_scheduling_work.csv
+  school: tour_scheduling_school.csv
+  univ: tour_scheduling_school.csv
 
 #CHOOSER_ORIG_COL_NAME: TAZ