Merge pull request #117 from urbansim/py3-staging

Python 3 wrap-up

.travis.yml

@@ -1,30 +1,25 @@
 language: python
-sudo: false
+
 python:
-- '2.7'
+  - '2.7'
+  - '3.6'
+
+# The Statsmodels line is to resolve a hopefully temporary problem --
+# Statsmodels 0.11 dropped support for Python 2.7, but as of 2020-02-23 is
+# being installed anyway by PyPI in Python 2.7 environments. If this isn't
+# fixed upstream, the best place to address it is in UrbanSim itself, where we
+# can make the Statsmodels version conditional on the Python version.
+
 install:
-- if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-    wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh;
-  else
-    wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
-  fi
-- bash miniconda.sh -b -p $HOME/miniconda
-- export PATH="$HOME/miniconda/bin:$PATH"
-- hash -r
-- conda config --set always_yes yes --set changeps1 no
-- conda update -q conda
-# Useful for debugging any issues with conda
-- conda info -a
-# don't think we need all these packages, but copying from urbansim
-- >
-  conda create -q -c synthicity -n test-environment
-  python=$TRAVIS_PYTHON_VERSION
-  cytoolz ipython-notebook jinja2 matplotlib numpy pandas patsy pip scipy
-  statsmodels pytables pytest pyyaml pandana
-- source activate test-environment
-- pip install pep8
-- pip install -r requirements.txt
+  - pip install statsmodels==0.10.*  # see comment above
+  - pip install .  # baus and requirements
+  - pip install coverage pycodestyle pytest  # testing tools
+  - pip list
+
 script:
-- pep8 baus
-- pep8 scripts
-- py.test baus
+  - pycodestyle baus
+  - pycodestyle scripts
+  - coverage run --source baus --module pytest --verbose
+
+after_success:
+  - coverage report

baus.py

@@ -24,7 +24,7 @@
 
 SLACK = MAPS = "URBANSIM_SLACK" in os.environ
 LOGS = True
-RANDOM_SEED = True
+RANDOM_SEED = False
 INTERACT = False
 SCENARIO = None
 MODE = "simulation"

baus/datasources.py

@@ -57,7 +57,7 @@ def building_type_map(mapping):
 def year():
     try:
         return orca.get_injectable("iter_var")
-    except:
+    except Exception as e:
         pass
     # if we're not running simulation, return base year
     return 2014
@@ -567,7 +567,7 @@ def get_logsum_file(type='mandatory'):
             else:
                 return orca.get_injectable('previous_{}_logsum_file'
                                            .format(type))
-    except:
+    except Exception as e:
         if 'logsum' in logsums:
             ls = logsums['logsum']
             ls_type = 'generic'

baus/output_csv_utils.py

@@ -223,15 +223,15 @@ def compare_outcome_for(variable, runs, set_geography):
 
 
 def subtract_base_year_urban_footprint(run_number):
-        base_year_filename = \
-            'runs/run{}_urban_footprint_summary_summaries_{}.csv'.\
-            format(run_number, 2010)
-        bdf = pd.read_csv(base_year_filename, index_col=0)
-        outcome_year_filename = \
-            'runs/run{}_urban_footprint_summary_summaries_{}.csv'.\
-            format(run_number, 2040)
-        odf = pd.read_csv(outcome_year_filename, index_col=0)
-        sdf = odf - bdf
-        sdf.to_csv(
-            'runs/run{}_urban_footprint_subtracted_summaries_{}.csv'
-            .format(run_number, 2040))
+    base_year_filename = \
+        'runs/run{}_urban_footprint_summary_summaries_{}.csv'.\
+        format(run_number, 2010)
+    bdf = pd.read_csv(base_year_filename, index_col=0)
+    outcome_year_filename = \
+        'runs/run{}_urban_footprint_summary_summaries_{}.csv'.\
+        format(run_number, 2040)
+    odf = pd.read_csv(outcome_year_filename, index_col=0)
+    sdf = odf - bdf
+    sdf.to_csv(
+        'runs/run{}_urban_footprint_subtracted_summaries_{}.csv'
+        .format(run_number, 2040))

baus/summaries.py

@@ -273,10 +273,10 @@ def policy_activated(policy_loc, policy_nm, scenario):
     counties = ["alameda", "contra_costa", "marin", "napa", "san_mateo",
                 "san_francisco", "santa_clara", "solano", "sonoma"]
     for county in counties:
-            policy_loc = (policy["acct_settings"]["lump_sum_accounts"]
-                          [county+"_bond_settings"]["enable_in_scenarios"])
-            if scenario in policy_loc:
-                counter += 1
+        policy_loc = (settings["acct_settings"]["lump_sum_accounts"]
+                      [county+"_bond_settings"]["enable_in_scenarios"])
+        if scenario in policy_loc:
+            counter += 1
     write("Affordable housing bonds are activated for %d counties" % counter)
 
     # workplace preferences are in the development projects list
@@ -346,7 +346,7 @@ def topsheet(households, jobs, buildings, parcels, zones, year,
 
     try:
         base_year_measures = orca.get_injectable("base_year_measures")
-    except:
+    except Exception as e:
         # the base year measures don't exist - we didn't run year 2010
         # this can happen when we skip the first year, usually because
         # we don't want to waste time doing so
@@ -534,7 +534,7 @@ def compare_to_targets(parcels, buildings, jobs, households, abag_targets,
 
     # compute correlection for pda and juris, for households and for jobs
 
-    l = []
+    li = []
 
     df = pd.DataFrame()
 
@@ -558,7 +558,7 @@ def compare_to_targets(parcels, buildings, jobs, households, abag_targets,
 
             assert len(abag_distribution) == len(baus_distribution)
 
-            l.append((geo, typ, abag_distribution.corr(baus_distribution)))
+            li.append((geo, typ, abag_distribution.corr(baus_distribution)))
 
     if write_comparison_dfs:
 
@@ -568,7 +568,7 @@ def compare_to_targets(parcels, buildings, jobs, households, abag_targets,
         df.to_csv(os.path.join("runs", "run%d_targets_comparison_%d.csv" %
                   (run_number, year)))
 
-    return l
+    return li
 
 
 @orca.step()
@@ -1772,7 +1772,7 @@ def write(s):
 
         try:
             slr_demolish_cum = orca.get_table("slr_demolish_cum").to_frame()
-        except:
+        except Exception as e:
             slr_demolish_cum = pd.DataFrame()
         slr_demolish = orca.get_table("slr_demolish").to_frame()
         slr_demolish_cum = slr_demolish.append(slr_demolish_cum)
@@ -1787,7 +1787,7 @@ def write(s):
         try:
             hh_unplaced_slr_cum = \
                 orca.get_table("hh_unplaced_slr_cum").to_frame()
-        except:
+        except Exception as e:
             hh_unplaced_slr_cum = pd.DataFrame()
         hh_unplaced_slr = orca.get_injectable("hh_unplaced_slr")
         hh_unplaced_slr_cum = hh_unplaced_slr.append(hh_unplaced_slr_cum)
@@ -1811,7 +1811,7 @@ def write(s):
         try:
             jobs_unplaced_slr_cum = \
                 orca.get_table("jobs_unplaced_slr_cum").to_frame()
-        except:
+        except Exception as e:
             jobs_unplaced_slr_cum = pd.DataFrame()
         jobs_unplaced_slr = orca.get_injectable("jobs_unplaced_slr")
         jobs_unplaced_slr_cum = jobs_unplaced_slr.append(jobs_unplaced_slr_cum)

baus/utils.py

@@ -48,7 +48,7 @@ def save_and_restore_state(in_d, outhdf="save_state.h5"):
     for table_name, table in in_d.items():
         try:
             table = table.local
-        except:
+        except Exception as e:
             # not a dataframe wrapper
             continue
         print("Saving", table_name)