Work in progress: ColumnFromBroadcast and ColumnFromAggregation

README.md

@@ -26,4 +26,4 @@ See the online documentation for much more: https://urbansim-templates.readthedo
 
 Some additional documentation is available within the repo in `CHANGELOG.md`, `CONTRIBUTING.md`, `/docs/README.md`, and `/tests/README.md`.
 
-There's discussion of current and planned features in the [Pull requests](https://github.com/udst/urbansim_templates/pulls?utf8=✓&q=is%3Apr) and [Issues](https://github.com/udst/urbansim_templates/issues?utf8=✓&q=is%3Aissue), both open and closed.
+There's discussion of current and planned features in the open and closed [Pull requests](https://github.com/udst/urbansim_templates/pulls?utf8=✓&q=is%3Apr) and [Issues](https://github.com/udst/urbansim_templates/issues?utf8=✓&q=is%3Aissue).

docs/source/data-templates.rst

@@ -13,7 +13,7 @@ Data templates help you load tables into `Orca <https://udst.github.io/orca>`__,
     t = LoadTable()
     t.table = 'buildings'  # a name for the Orca table
     t.source_type = 'csv'
-    t.path = 'buildings.csv'
+    t.path = '../data/buildings.csv'
     t.csv_index_cols = 'building_id'
     t.name = 'load_buildings'  # a name for the model step that sets up the table
 
@@ -35,7 +35,7 @@ Registration does two things: (a) it saves the configured template to disk as a
     import orca
     orca.run(['load_buildings'])
 
-Strictly speaking, running the model step doesn't load the data, it just sets up an Orca table with instructions for loading the data when it's needed. (This is called lazy evaluation.)
+Strictly speaking, running the template/step doesn't load the data, it just sets up an Orca table with instructions for loading the data when it's needed. (This is called lazy evaluation.)
 
 .. code-block:: python
 

docs/source/getting-started.rst

@@ -18,9 +18,7 @@ UrbanSim Templates was created in 2018 by Sam Maurer ([email protected]), who
 Installation
 ------------
 
-UrbanSim Templates is tested with Python versions 2.7, 3.5, 3.6, and 3.7. 
-
-As of Feb. 2019, there is an installation problem in Python 3.7 when using Pip (because of an issue with Orca's PyTables dependency). Conda should work.
+UrbanSim Templates is tested with Python versions 2.7, 3.5, 3.6, and 3.7, across Mac, Linux, and Windows. Support for Python 2.7 will probably be removed in an upcoming release. 
 
 .. note::
     It can be helpful to set up a dedicated Python environment for each project you work on. This lets you use a stable and replicable set of libraries that won't be affected by other projects. Here are some good `environment settings <https://gist.github.com/smmaurer/f3a4f424a4aa877fb73e1cb2567bd89d>`__ for UrbanSim Templates projects.

tests/pytest.ini

@@ -4,6 +4,8 @@ filterwarnings =
     ignore:::urbansim
     ignore:::pandas
     ignore:::past
+    ignore:::patsy
     ignore:::prettytable
+    ignore:::scipy
     ignore:::statsmodels
     ignore:::yaml

tests/test_binary_logit.py

@@ -10,6 +10,8 @@
 
 @pytest.fixture
 def orca_session():
+    orca.clear_all()
+
     d1 = {'a': np.random.random(100),
           'b': np.random.randint(2, size=100)}
 

tests/test_column_broadcast.py

@@ -0,0 +1,213 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+import orca
+
+from urbansim_templates import modelmanager
+from urbansim_templates.data import ColumnFromBroadcast
+from urbansim_templates.utils import validate_template
+
+
+@pytest.fixture
+def orca_session():
+    """
+    Set up a clean Orca and ModelManager session, with data tables.
+
+    """
+    orca.clear_all()
+    modelmanager.initialize()
+
+    d = {'building_id': [1,2,3,4], 'value': [4,4,4,4]}
+    df = pd.DataFrame(d).set_index('building_id')
+    orca.add_table('buildings', df)
+
+    d = {'household_id': [1,2,3], 'building_id': [2,3,4]}
+    df = pd.DataFrame(d).set_index('household_id')
+    orca.add_table('households', df)
+
+
+
+###############################
+## BEHAVIOR OF TEMPLATE
+
+
+
+
+
+
+
+
+
+
+
+# def test_expression(orca_session):
+#     """
+#     Check that column is created and expression evaluated correctly.
+#     
+#     """
+#     c = ColumnFromExpression()
+#     c.column_name = 'c'
+#     c.table = 'obs'
+#     c.expression = 'a * 5 + sqrt(b)'
+#     
+#     c.run()
+#     
+#     val1 = orca.get_table('obs').get_column('c')
+#     df = orca.get_table('obs').to_frame()
+#     val2 = df.a * 5 + np.sqrt(df.b)
+#     assert(val1.equals(val2))
+#     
+# 
+# def test_data_type(orca_session):
+#     """
+#     Check that casting data type works.
+#     
+#     """
+#     orca.add_table('tab', pd.DataFrame({'a': [0.1, 1.33, 2.4]}))
+#     
+#     c = ColumnFromExpression()
+#     c.column_name = 'b'
+#     c.table = 'tab'
+#     c.expression = 'a'
+#     c.run()
+#     
+#     v1 = orca.get_table('tab').get_column('b').values
+#     np.testing.assert_equal(v1, [0.1, 1.33, 2.4])
+#     
+#     c.data_type = 'int'
+#     c.run()
+# 
+#     v1 = orca.get_table('tab').get_column('b').values
+#     np.testing.assert_equal(v1, [0, 1, 2])
+# 
+# 
+# def test_missing_values(orca_session):
+#     """
+#     Check that filling in missing values works.
+#     
+#     """
+#     orca.add_table('tab', pd.DataFrame({'a': [0.1, np.nan, 2.4]}))
+#     
+#     c = ColumnFromExpression()
+#     c.column_name = 'b'
+#     c.table = 'tab'
+#     c.expression = 'a'
+#     c.run()
+#     
+#     v1 = orca.get_table('tab').get_column('b').values
+#     np.testing.assert_equal(v1, [0.1, np.nan, 2.4])
+#     
+#     c.missing_values = 5
+#     c.run()
+# 
+#     v1 = orca.get_table('tab').get_column('b').values
+#     np.testing.assert_equal(v1, [0.1, 5.0, 2.4])
+# 
+# 
+# def test_modelmanager_registration(orca_session):
+#     """
+#     Check that modelmanager registration and auto-run work as expected.
+#     
+#     """
+#     c = ColumnFromExpression()
+#     c.column_name = 'c'
+#     c.table = 'obs'
+#     c.expression = 'a + b'
+#     
+#     modelmanager.register(c)
+#     modelmanager.remove_step(c.name)
+#     assert('c' in orca.get_table('obs').columns)
+# 
+# 
+# def test_expression_with_standalone_columns(orca_session):
+#     """
+#     Check that expression can assemble data from stand-alone columns that are not part 
+#     of the core DataFrame wrapped by a table.
+#     
+#     """
+#     c = ColumnFromExpression()
+#     c.column_name = 'c'
+#     c.table = 'obs'
+#     c.expression = 'a + b'
+#     
+#     modelmanager.register(c)
+#     modelmanager.remove_step(c.name)
+# 
+#     d = ColumnFromExpression()
+#     d.column_name = 'd'
+#     d.table = 'obs'
+#     d.expression = 'a + c'
+#     
+#     d.run()
+#     assert('d' in orca.get_table('obs').columns)
+# 
+
+
+
+
+###############################
+## SPEC CONFORMANCE AND BAD DATA INPUTS
+
+def test_template_validity():
+    """
+    Check template conforms to basic spec.
+
+    """
+    assert validate_template(ColumnFromBroadcast)
+
+
+# def test_missing_colname(orca_session):
+#     """
+#     Missing column_name should raise a ValueError.
+#     
+#     """
+#     c = ColumnFromExpression()
+#     c.table = 'tab'
+#     c.expression = 'a'
+#     
+#     try:
+#         c.run()
+#     except ValueError as e:
+#         print(e)
+#         return
+#     
+#     pytest.fail()
+# 
+# 
+# def test_missing_table(orca_session):
+#     """
+#     Missing table should raise a ValueError.
+#     
+#     """
+#     c = ColumnFromExpression()
+#     c.column_name = 'col'
+#     c.expression = 'a'
+#     
+#     try:
+#         c.run()
+#     except ValueError as e:
+#         print(e)
+#         return
+#     
+#     pytest.fail()
+# 
+# 
+# def test_missing_expression(orca_session):
+#     """
+#     Missing expression should raise a ValueError.
+#     
+#     """
+#     c = ColumnFromExpression()
+#     c.column_name = 'col'
+#     c.table = 'tab'
+#     
+#     try:
+#         c.run()
+#     except ValueError as e:
+#         print(e)
+#         return
+#     
+#     pytest.fail()
+# 
+# 

tests/test_large_multinomial_logit.py

@@ -12,6 +12,8 @@
 
 @pytest.fixture
 def orca_session():
+    orca.clear_all()
+
     d1 = {'oid': np.arange(10), 
           'obsval': np.random.random(10),
           'choice': np.random.choice(np.arange(20), size=10)}

tests/test_regression.py

@@ -10,6 +10,8 @@
 
 @pytest.fixture
 def orca_session():
+    orca.clear_all()
+
     d1 = {'a': np.random.random(100),
           'b': np.random.random(100)}
 

tests/test_segmented_large_multinomial_logit.py

@@ -16,6 +16,8 @@ def orca_session():
     Set up a clean Orca session with a couple of data tables.
 
     """
+    orca.clear_all()
+
     d1 = {'oid': np.arange(100), 
           'group': np.random.choice(['A','B','C'], size=100),
           'int_group': np.random.choice([3,4], size=100),

tests/test_small_multinomial_logit.py

@@ -11,6 +11,8 @@
 
 @pytest.fixture
 def orca_session():
+    orca.clear_all()
+
     d1 = {'id': np.arange(100),
           'building_id': np.arange(100),
           'a': np.random.random(100),

urbansim_templates/data/__init__.py

@@ -1,3 +1,4 @@
-from .column_from_expression import ColumnFromExpression, ExpressionSettings
-from .load_table import LoadTable
-from .save_table import SaveTable
+from .column_broadcast import ColumnFromBroadcast
+from .column_expression import ColumnFromExpression, ExpressionSettings
+from .table_load import LoadTable
+from .table_save import SaveTable