diff --git a/activitysim/abm/models/atwork_subtour_frequency.py b/activitysim/abm/models/atwork_subtour_frequency.py index 7bbee371f..3579d2ed7 100644 --- a/activitysim/abm/models/atwork_subtour_frequency.py +++ b/activitysim/abm/models/atwork_subtour_frequency.py @@ -15,6 +15,7 @@ simulate, tracing, workflow, + asim_enum, ) logger = logging.getLogger(__name__) @@ -41,7 +42,7 @@ def atwork_subtour_frequency( trace_label = "atwork_subtour_frequency" model_settings_file_name = "atwork_subtour_frequency.yaml" trace_hh_id = state.settings.trace_hh_id - work_tours = tours[tours.tour_type == "work"] + work_tours = tours[tours.tour_type == asim_enum.TourPurpose.work] # - if no work_tours if len(work_tours) == 0: @@ -115,7 +116,7 @@ def atwork_subtour_frequency( state.add_table("tours", tours) # - create atwork_subtours based on atwork_subtour_frequency choice names - work_tours = tours[tours.tour_type == "work"] + work_tours = tours[tours.tour_type == asim_enum.TourPurpose.work] assert not work_tours.atwork_subtour_frequency.isnull().any() subtours = process_atwork_subtours(state, work_tours, alternatives) diff --git a/activitysim/abm/models/joint_tour_composition.py b/activitysim/abm/models/joint_tour_composition.py index 50c458e7a..6aebd938c 100644 --- a/activitysim/abm/models/joint_tour_composition.py +++ b/activitysim/abm/models/joint_tour_composition.py @@ -14,6 +14,7 @@ simulate, tracing, workflow, + asim_enum, ) logger = logging.getLogger(__name__) @@ -21,7 +22,7 @@ def add_null_results(state, trace_label, tours): logger.info("Skipping %s: add_null_results" % trace_label) - tours["composition"] = "" + tours["composition"] = asim_enum.TourComposition.na state.add_table("tours", tours) @@ -38,7 +39,7 @@ def joint_tour_composition( trace_label = "joint_tour_composition" model_settings_file_name = "joint_tour_composition.yaml" - joint_tours = tours[tours.tour_category == "joint"] + joint_tours = tours[tours.tour_category == asim_enum.TourCategory.joint] # - if no joint tours if joint_tours.shape[0] == 0: @@ -63,6 +64,7 @@ def joint_tour_composition( locals_dict = { "persons": persons, "hh_time_window_overlap": lambda *x: hh_time_window_overlap(state, *x), + "asim_enum": asim_enum, } expressions.assign_columns( @@ -106,6 +108,7 @@ def joint_tour_composition( # convert indexes to alternative names choices = pd.Series(model_spec.columns[choices.values], index=choices.index) + choices = choices.map(asim_enum.TourComposition.__dict__) if estimator: estimator.write_choices(choices) @@ -117,7 +120,7 @@ def joint_tour_composition( joint_tours["composition"] = choices # reindex since we ran model on a subset of households - tours["composition"] = choices.reindex(tours.index).fillna("").astype(str) + tours["composition"] = choices.reindex(tours.index).fillna(asim_enum.TourComposition.na) state.add_table("tours", tours) tracing.print_summary( diff --git a/activitysim/abm/models/joint_tour_frequency.py b/activitysim/abm/models/joint_tour_frequency.py index 66996ce1d..1849b00ae 100644 --- a/activitysim/abm/models/joint_tour_frequency.py +++ b/activitysim/abm/models/joint_tour_frequency.py @@ -16,6 +16,7 @@ simulate, tracing, workflow, + asim_enum, ) logger = logging.getLogger(__name__) @@ -40,6 +41,9 @@ def joint_tour_frequency( alternatives = simulate.read_model_alts( state, "joint_tour_frequency_alternatives.csv", set_index="alt" ) + # add "j" in front of joint tour alternatives + alternatives.index = "j" + alternatives.index.to_series() + alternatives.index = alternatives.index.to_series().map(asim_enum.JointTourFrequency.__dict__) # - only interested in households with more than one cdap travel_active person and # - at least one non-preschooler @@ -60,6 +64,7 @@ def joint_tour_frequency( locals_dict = { "persons": persons, "hh_time_window_overlap": lambda *x: hh_time_window_overlap(state, *x), + "asim_enum": asim_enum, } expressions.assign_columns( @@ -98,6 +103,9 @@ def joint_tour_frequency( # convert indexes to alternative names choices = pd.Series(model_spec.columns[choices.values], index=choices.index) + # add "j" in front of joint tour alternatives + choices = "j" + choices + choices = choices.map(asim_enum.JointTourFrequency.__dict__) if estimator: estimator.write_choices(choices) @@ -130,7 +138,7 @@ def joint_tour_frequency( # we expect there to be an alt with no tours - which we can use to backfill non-travelers no_tours_alt = (alternatives.sum(axis=1) == 0).index[0] households["joint_tour_frequency"] = ( - choices.reindex(households.index).fillna(no_tours_alt).astype(str) + choices.reindex(households.index).fillna(asim_enum.JointTourFrequency.na) ) households["num_hh_joint_tours"] = ( @@ -156,7 +164,7 @@ def joint_tour_frequency( if estimator: survey_tours = estimation.manager.get_survey_table("tours") - survey_tours = survey_tours[survey_tours.tour_category == "joint"] + survey_tours = survey_tours[survey_tours.tour_category == asim_enum.TourCategory.joint] print(f"len(survey_tours) {len(survey_tours)}") print(f"len(joint_tours) {len(joint_tours)}") diff --git a/activitysim/abm/models/joint_tour_participation.py b/activitysim/abm/models/joint_tour_participation.py index 066ef703c..c9a34dd35 100644 --- a/activitysim/abm/models/joint_tour_participation.py +++ b/activitysim/abm/models/joint_tour_participation.py @@ -17,6 +17,7 @@ simulate, tracing, workflow, + asim_enum, ) from activitysim.core.util import assign_in_place, reindex @@ -49,8 +50,8 @@ def joint_tour_participation_candidates(joint_tours, persons_merged): # - filter out ineligible candidates (adults for children-only tours, and vice-versa) eligible = ~( - ((candidates.composition == "adults") & ~candidates.adult) - | ((candidates.composition == "children") & candidates.adult) + ((candidates.composition == asim_enum.TourComposition.adults) & ~candidates.adult) + | ((candidates.composition == asim_enum.TourComposition.children) & candidates.adult) ) candidates = candidates[eligible] @@ -79,8 +80,8 @@ def get_tour_satisfaction(candidates, participate): candidates = candidates[participate] # if this happens, we would need to filter them out! - assert not ((candidates.composition == "adults") & ~candidates.adult).any() - assert not ((candidates.composition == "children") & candidates.adult).any() + assert not ((candidates.composition == asim_enum.TourComposition.adults) & ~candidates.adult).any() + assert not ((candidates.composition == asim_enum.TourComposition.children) & candidates.adult).any() # FIXME tour satisfaction - hack # annotate_households_cdap.csv says there has to be at least one non-preschooler in household @@ -104,8 +105,8 @@ def get_tour_satisfaction(candidates, participate): # (x.composition == 'children') & (x.participants > 1) & (x.preschoolers < x.participants) | \ # (x.composition == 'mixed') & (x.adults > 0) & (x.participants > x.adults) - satisfaction = (x.composition != "mixed") & (x.participants > 1) | ( - x.composition == "mixed" + satisfaction = (x.composition != asim_enum.TourComposition.mixed) & (x.participants > 1) | ( + x.composition == asim_enum.TourComposition.mixed ) & (x.adults > 0) & (x.participants > x.adults) satisfaction = satisfaction.reindex(tour_ids).fillna(False).astype(bool) diff --git a/activitysim/abm/models/mandatory_scheduling.py b/activitysim/abm/models/mandatory_scheduling.py index 2e5401bf1..2533ffb06 100644 --- a/activitysim/abm/models/mandatory_scheduling.py +++ b/activitysim/abm/models/mandatory_scheduling.py @@ -8,7 +8,7 @@ from activitysim.abm.models.util.tour_scheduling import run_tour_scheduling from activitysim.core import timetable as tt -from activitysim.core import tracing, workflow +from activitysim.core import tracing, workflow, asim_enum from activitysim.core.util import assign_in_place, reindex logger = logging.getLogger(__name__) @@ -30,7 +30,7 @@ def mandatory_tour_scheduling( model_name = "mandatory_tour_scheduling" trace_label = model_name - mandatory_tours = tours[tours.tour_category == "mandatory"] + mandatory_tours = tours[tours.tour_category == asim_enum.TourCategory.mandatory] # - if no mandatory_tours if mandatory_tours.shape[0] == 0: @@ -46,11 +46,11 @@ def mandatory_tour_scheduling( # we conflate them by segmenting tour processing to align with primary_purpose tour_segment_col = "mandatory_tour_seg" assert tour_segment_col not in mandatory_tours - is_university_tour = (mandatory_tours.tour_type == "school") & reindex( + is_university_tour = (mandatory_tours.tour_type == asim_enum.TourPurpose.school) & reindex( persons_merged.is_university, mandatory_tours.person_id ) mandatory_tours[tour_segment_col] = mandatory_tours.tour_type.where( - ~is_university_tour, "univ" + ~is_university_tour, asim_enum.TourPurpose.univ.value ) choices = run_tour_scheduling( @@ -66,7 +66,7 @@ def mandatory_tour_scheduling( state.add_table("tours", tours) # updated df for tracing - mandatory_tours = tours[tours.tour_category == "mandatory"] + mandatory_tours = tours[tours.tour_category == asim_enum.TourCategory.mandatory] state.tracing.dump_df( DUMP, diff --git a/activitysim/abm/models/mandatory_tour_frequency.py b/activitysim/abm/models/mandatory_tour_frequency.py index 166c671e0..ff2cf846c 100644 --- a/activitysim/abm/models/mandatory_tour_frequency.py +++ b/activitysim/abm/models/mandatory_tour_frequency.py @@ -14,6 +14,7 @@ simulate, tracing, workflow, + asim_enum, ) logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ def add_null_results(state, trace_label, mandatory_tour_frequency_settings): logger.info("Skipping %s: add_null_results", trace_label) persons = state.get_dataframe("persons") - persons["mandatory_tour_frequency"] = "" + persons["mandatory_tour_frequency"] = asim_enum.MandatoryTourFrequency.na.value tours = pd.DataFrame() tours["tour_category"] = None @@ -110,6 +111,7 @@ def mandatory_tour_frequency( # convert indexes to alternative names choices = pd.Series(model_spec.columns[choices.values], index=choices.index) + choices = choices.map(asim_enum.MandatoryTourFrequency._member_map_) if estimator: estimator.write_choices(choices) @@ -128,6 +130,8 @@ def mandatory_tour_frequency( alternatives = simulate.read_model_alts( state, "mandatory_tour_frequency_alternatives.csv", set_index="alt" ) + # change the alt index to asim_enum + alternatives.index = alternatives.index.to_series().map(asim_enum.MandatoryTourFrequency._member_map_) choosers["mandatory_tour_frequency"] = choices.reindex(choosers.index) mandatory_tours = process_mandatory_tours( @@ -143,13 +147,14 @@ def mandatory_tour_frequency( # need to reindex as we only handled persons with cdap_activity == 'M' persons["mandatory_tour_frequency"] = ( - choices.reindex(persons.index).fillna("").astype(str) + choices.reindex(persons.index).fillna(asim_enum.MandatoryTourFrequency.na.value) ) expressions.assign_columns( state, df=persons, model_settings=model_settings.get("annotate_persons"), + locals_dict={"asim_enum":asim_enum}, trace_label=tracing.extend_trace_label(trace_label, "annotate_persons"), ) diff --git a/activitysim/abm/models/non_mandatory_tour_frequency.py b/activitysim/abm/models/non_mandatory_tour_frequency.py index 86a794102..b8c21b805 100644 --- a/activitysim/abm/models/non_mandatory_tour_frequency.py +++ b/activitysim/abm/models/non_mandatory_tour_frequency.py @@ -21,6 +21,7 @@ simulate, tracing, workflow, + asim_enum, ) from activitysim.core.interaction_simulate import interaction_simulate @@ -177,7 +178,7 @@ def non_mandatory_tour_frequency( # - preprocessor preprocessor_settings = model_settings.get("preprocessor", None) if preprocessor_settings: - locals_dict = {"person_max_window": lambda x: person_max_window(state, x)} + locals_dict = {"person_max_window": lambda x: person_max_window(state, x), "asim_enum":asim_enum} expressions.assign_columns( state, diff --git a/activitysim/abm/models/util/canonical_ids.py b/activitysim/abm/models/util/canonical_ids.py index 0ca1a1d3b..28963dcc2 100644 --- a/activitysim/abm/models/util/canonical_ids.py +++ b/activitysim/abm/models/util/canonical_ids.py @@ -8,7 +8,7 @@ import numpy as np import pandas as pd -from activitysim.core import simulate, workflow +from activitysim.core import simulate, workflow, asim_enum logger = logging.getLogger(__name__) @@ -383,7 +383,7 @@ def set_tour_index( assert tour_num_col in tours.columns # create string tour_id corresonding to keys in possible_tours (e.g. 'work1', 'j_shopping2') - tours["tour_id"] = tours.tour_type + tours[tour_num_col].map(str) + tours["tour_id"] = tours.tour_type.apply(lambda x: asim_enum.TourPurpose._value2member_map_[x].name) + tours[tour_num_col].map(str) if parent_tour_num_col: # we need to distinguish between subtours of different work tours diff --git a/activitysim/abm/models/util/mode.py b/activitysim/abm/models/util/mode.py index 8a75ae8b6..0397c4f7f 100644 --- a/activitysim/abm/models/util/mode.py +++ b/activitysim/abm/models/util/mode.py @@ -8,7 +8,7 @@ import pandas as pd -from activitysim.core import config, expressions, simulate, workflow +from activitysim.core import config, expressions, simulate, workflow, asim_enum from activitysim.core.estimation import Estimator """ @@ -110,7 +110,7 @@ def run_tour_mode_choice_simulate( spec = state.filesystem.read_model_spec(file_name=model_settings["SPEC"]) coefficients = state.filesystem.get_segment_coefficients( - model_settings, tour_purpose + model_settings, asim_enum.TourPurpose._value2member_map_[tour_purpose].name ) spec = simulate.eval_coefficients(state, spec, coefficients, estimator) diff --git a/activitysim/abm/models/util/tour_frequency.py b/activitysim/abm/models/util/tour_frequency.py index 9c9d8a567..4bac0822e 100644 --- a/activitysim/abm/models/util/tour_frequency.py +++ b/activitysim/abm/models/util/tour_frequency.py @@ -8,7 +8,7 @@ import pandas as pd from activitysim.abm.models.util.canonical_ids import set_tour_index -from activitysim.core import config, workflow +from activitysim.core import config, workflow, asim_enum from activitysim.core.util import reindex logger = logging.getLogger(__name__) @@ -57,6 +57,8 @@ def create_tours(tour_counts, tour_category, parent_col="person_id"): # reformat with the columns given below tours = tour_counts.stack().reset_index() tours.columns = [parent_col, "tour_type", "tour_type_count"] + # convert tour type to asim_enum + tours["tour_type"] = tours["tour_type"].map(asim_enum.TourPurpose._member_map_) """ tour_type tour_type_count @@ -215,7 +217,7 @@ def process_mandatory_tours( tours = process_tours( persons.mandatory_tour_frequency.dropna(), mandatory_tour_frequency_alts, - tour_category="mandatory", + tour_category=asim_enum.TourCategory.mandatory.value, ) tours_merged = pd.merge( @@ -228,7 +230,7 @@ def process_mandatory_tours( # by default work tours are first for work_and_school tours # swap tour_nums for non-workers so school tour is 1 and work is 2 work_and_school_and_student = ( - tours_merged.mandatory_tour_frequency == "work_and_school" + tours_merged.mandatory_tour_frequency == asim_enum.MandatoryTourFrequency.work_and_school ) & ~tours_merged.is_worker tours.tour_num = tours.tour_num.where( @@ -237,7 +239,7 @@ def process_mandatory_tours( # work tours destination is workplace_zone_id, school tours destination is school_zone_id tours["destination"] = tours_merged.workplace_zone_id.where( - (tours_merged.tour_type == "work"), tours_merged.school_zone_id + (tours_merged.tour_type == asim_enum.TourPurpose.work), tours_merged.school_zone_id ) tours["origin"] = tours_merged.home_zone_id @@ -292,7 +294,7 @@ def process_non_mandatory_tours(state: workflow.State, persons, tour_counts): column names of the alternatives DataFrame supplied above. """ - tours = create_tours(tour_counts, tour_category="non_mandatory") + tours = create_tours(tour_counts, tour_category=asim_enum.TourCategory.non_mandatory.value) tours["household_id"] = reindex(persons.household_id, tours.person_id) tours["origin"] = reindex(persons.home_zone_id, tours.person_id) @@ -362,7 +364,7 @@ def process_atwork_subtours( tours = process_tours( work_tours.atwork_subtour_frequency.dropna(), atwork_subtour_frequency_alts, - tour_category="atwork", + tour_category=asim_enum.TourCategory.atwork.value, parent_col=parent_col, ) @@ -446,7 +448,7 @@ def process_joint_tours( tours = process_tours( joint_tour_frequency.dropna(), joint_tour_frequency_alts, - tour_category="joint", + tour_category=asim_enum.TourCategory.joint.value, parent_col="household_id", ) @@ -515,7 +517,7 @@ def process_joint_tours_frequency_composition( state, joint_tour_frequency_composition.dropna(), joint_tour_frequency_composition_alts, - tour_category="joint", + tour_category=asim_enum.TourCategory.joint.value, parent_col="household_id", ) @@ -693,6 +695,8 @@ def create_joint_tours( tours_purp.columns = [parent_col, "tour_id_temp", "tour_type"] tours_purp["tour_id_temp"] = range(1, 1 + len(tours_purp)) tours_purp["tour_type"] = tours_purp["tour_type"].map(tour_type_dict) + # convert tour type to asim_enum + tours_purp["tour_type"] = tours_purp["tour_type"].map(asim_enum.TourPurpose._member_map_) """ tour_id_temp tour_type @@ -711,6 +715,8 @@ def create_joint_tours( tours_comp.columns = [parent_col, "tour_id_temp", "composition"] tours_comp["tour_id_temp"] = range(1, 1 + len(tours_comp)) tours_comp["composition"] = tours_comp["composition"].map(tour_comp_dict) + # convert tour type to asim_enum + tours_comp["composition"] = tours_comp["composition"].map(asim_enum.TourComposition._member_map_) """ tour_id_temp tour_composition @@ -750,7 +756,7 @@ def create_joint_tours( """ # set these here to ensure consistency across different tour categories - assert tour_category in ["mandatory", "non_mandatory", "atwork", "joint"] + assert tour_category in asim_enum.TourCategory tours["tour_category"] = tour_category # for joint tours, the correct number will be filled in after participation step diff --git a/activitysim/abm/models/util/vectorize_tour_scheduling.py b/activitysim/abm/models/util/vectorize_tour_scheduling.py index 775d84b7b..cd11c236e 100644 --- a/activitysim/abm/models/util/vectorize_tour_scheduling.py +++ b/activitysim/abm/models/util/vectorize_tour_scheduling.py @@ -9,7 +9,7 @@ from activitysim.core import chunk, config, expressions, los, simulate from activitysim.core import timetable as tt -from activitysim.core import tracing, workflow +from activitysim.core import tracing, workflow, asim_enum from activitysim.core.interaction_sample_simulate import interaction_sample_simulate from activitysim.core.util import reindex @@ -992,7 +992,7 @@ def vectorize_tour_scheduling( ) nth_tours_in_segment = nth_tours[ - nth_tours[tour_segment_col] == tour_segment_name + nth_tours[tour_segment_col] == asim_enum.TourPurpose[tour_segment_name] ] if nth_tours_in_segment.empty: logger.info("skipping empty segment %s" % tour_segment_name) diff --git a/activitysim/core/asim_enum.py b/activitysim/core/asim_enum.py new file mode 100644 index 000000000..dc84b4e3a --- /dev/null +++ b/activitysim/core/asim_enum.py @@ -0,0 +1,82 @@ +from enum import IntEnum + +# enum class for the different time periods +class TimePeriod(IntEnum): + EA = 1 + AM = 2 + MD = 3 + PM = 4 + NT = 5 + + +# enum class for mandatory tour frequency +class MandatoryTourFrequency(IntEnum): + na = 0 + work1 = 1 + work2 = 2 + school1 = 3 + school2 = 4 + work_and_school = 5 + + +# enum class for the different tour purposes +class TourPurpose(IntEnum): + work = 1 + school = 2 + univ = 3 + escort = 4 + shopping = 5 + othmaint = 6 + eatout = 7 + social = 8 + othdiscr = 9 + atwork = 10 + + +# enum class for the different tour category +class TourCategory(IntEnum): + mandatory = 1 + joint = 2 + non_mandatory = 3 + atwork = 4 + + +# enum class for the different daily activity pattern +class CdapCategory(IntEnum): + M = 1 + N = 2 + H = 3 + + +# enum class for joint tour frequency +class JointTourFrequency(IntEnum): + na = 0 + j0_tours = 1 + j1_Shop = 2 + j1_Main = 3 + j1_Eat = 4 + j1_Visit = 5 + j1_Disc = 6 + j2_SS = 7 + j2_SM = 8 + j2_SE = 9 + j2_SV = 10 + j2_SD = 11 + j2_MM = 12 + j2_ME = 13 + j2_MV = 14 + j2_MD = 15 + j2_EE = 16 + j2_EV = 17 + j2_ED = 18 + j2_VV = 19 + j2_VD = 20 + j2_DD = 21 + + +# enum class for tour composition +class TourComposition(IntEnum): + na = 0 + adults = 1 + children = 2 + mixed = 3 \ No newline at end of file diff --git a/activitysim/core/assign.py b/activitysim/core/assign.py index d504d6378..20a58ff1f 100644 --- a/activitysim/core/assign.py +++ b/activitysim/core/assign.py @@ -9,7 +9,7 @@ import numpy as np import pandas as pd -from activitysim.core import chunk, util, workflow +from activitysim.core import chunk, util, workflow, asim_enum logger = logging.getLogger(__name__) @@ -263,6 +263,7 @@ def to_series(x): _locals_dict[df_alias] = df else: _locals_dict["df"] = df + _locals_dict.update({"asim_enum":asim_enum}) local_keys = list(_locals_dict.keys()) # build a dataframe of eval results for non-temp targets diff --git a/activitysim/core/expressions.py b/activitysim/core/expressions.py index b57eca94c..72a77a7f2 100644 --- a/activitysim/core/expressions.py +++ b/activitysim/core/expressions.py @@ -3,8 +3,10 @@ from __future__ import annotations import logging +import io +import os -from . import assign, config, simulate, tracing, workflow +from . import assign, config, simulate, tracing, workflow, asim_enum from .util import assign_in_place, parse_suffix_args, suffix_expressions_df_str logger = logging.getLogger(__name__) @@ -102,6 +104,7 @@ def compute_columns(state, df, model_settings, locals_dict={}, trace_label=None) _locals_dict = assign.local_utilities(state) _locals_dict.update(locals_dict) _locals_dict.update(tables) + _locals_dict.update({"asim_enum":asim_enum}) # FIXME a number of asim model preprocessors want skim_dict - should they request it in model_settings.TABLES? try: @@ -153,6 +156,35 @@ def assign_columns( results = compute_columns(state, df, model_settings, locals_dict, trace_label) + buffer = io.StringIO() + results.info(memory_usage = 'deep', buf=buffer) + s = buffer.getvalue() + with open(os.path.join(state.filesystem.output_dir, trace_label+".assign_columns.results.info.txt"), "w", encoding="utf-8") as f: + f.write(s) + + results.memory_usage(deep = True).to_csv( + os.path.join(state.filesystem.output_dir, trace_label+".assign_columns.results.memory_usage_deep.txt") + ) + + results.memory_usage().to_csv( + os.path.join(state.filesystem.output_dir, trace_label+".assign_columns.results.memory_usage.txt") + ) + + + buffer = io.StringIO() + df.info(memory_usage = 'deep', buf=buffer) + s = buffer.getvalue() + with open(os.path.join(state.filesystem.output_dir, trace_label+".assign_columns.df.info.txt"), "w", encoding="utf-8") as f: + f.write(s) + + df.memory_usage(deep = True).to_csv( + os.path.join(state.filesystem.output_dir, trace_label+".assign_columns.df.memory_usage_deep.txt") + ) + + df.memory_usage().to_csv( + os.path.join(state.filesystem.output_dir, trace_label+".assign_columns.df.memory_usage.txt") + ) + assign_in_place(df, results) diff --git a/activitysim/core/interaction_simulate.py b/activitysim/core/interaction_simulate.py index 88dbfc73d..6d5d8da73 100644 --- a/activitysim/core/interaction_simulate.py +++ b/activitysim/core/interaction_simulate.py @@ -12,6 +12,10 @@ import numpy as np import pandas as pd +import psutil +import os +import io + from . import chunk, config, logit, simulate, tracing, workflow logger = logging.getLogger(__name__) @@ -76,6 +80,25 @@ def eval_interaction_utilities( trace_label = tracing.extend_trace_label(trace_label, "eval_interaction_utils") logger.info("Running eval_interaction_utilities on %s rows" % df.shape[0]) + logger.info("Writing out interaction df info") + + buffer = io.StringIO() + df.info(memory_usage = 'deep', buf=buffer) + s = buffer.getvalue() + with open(os.path.join(state.filesystem.output_dir, trace_label+".interaction_df.info.txt"), "w", encoding="utf-8") as f: + f.write(s) + + df.memory_usage(deep = True).to_csv( + os.path.join(state.filesystem.output_dir, trace_label+".interaction_df.memory_usage_deep.txt") + ) + + df.memory_usage().to_csv( + os.path.join(state.filesystem.output_dir, trace_label+".interaction_df.memory_usage.txt") + ) + + process = psutil.Process(os.getpid()) + logger.info("PID RSS when writing out interaction df info: %s" % process.memory_info().rss) + sharrow_enabled = state.settings.sharrow if locals_d is not None and locals_d.get("_sharrow_skip", False): diff --git a/activitysim/core/timetable.py b/activitysim/core/timetable.py index 85ecc6954..a86e4aeb4 100644 --- a/activitysim/core/timetable.py +++ b/activitysim/core/timetable.py @@ -9,7 +9,7 @@ import numpy as np import pandas as pd -from activitysim.core import chunk, configuration, workflow +from activitysim.core import chunk, configuration, workflow, asim_enum logger = logging.getLogger(__name__) @@ -253,7 +253,7 @@ def tour_map(persons, tours, tdd_alts, persons_id_col="person_id"): for keys, nth_tours in tours.groupby(["tour_type", "tour_type_num"], sort=True): tour_type = keys[0] - tour_sigil = sigil[tour_type] + tour_sigil = sigil[asim_enum.TourPurpose._value2member_map_[tour_type].name] # numpy array with one time window row for each row in nth_tours tour_windows = window_periods_df.loc[nth_tours.tdd].values diff --git a/activitysim/examples/prototype_mtc/configs/annotate_persons_mtf.csv b/activitysim/examples/prototype_mtc/configs/annotate_persons_mtf.csv index 00e0de101..a90e7afdb 100644 --- a/activitysim/examples/prototype_mtc/configs/annotate_persons_mtf.csv +++ b/activitysim/examples/prototype_mtc/configs/annotate_persons_mtf.csv @@ -2,7 +2,9 @@ Description,Target,Expression #,, annotate persons table after mandatory_tour_frequency model has run ,_PERSON_TOUR_COUNT,"lambda exp, persons, tours: tours.query(exp).groupby('person_id').size().reindex(persons.index).fillna(0).astype(np.int8)" ,_Q,lambda s: "'{}'".format(s) -work_and_school_and_worker,work_and_school_and_worker,(persons.mandatory_tour_frequency == 'work_and_school') & persons.is_worker -work_and_school_and_student,work_and_school_and_student,(persons.mandatory_tour_frequency == 'work_and_school') & persons.is_student -number of mandatory tours for each person,num_mand,"_PERSON_TOUR_COUNT('tour_category==%s' % _Q('mandatory'), persons, tours).fillna(0)" -number of work tours for each person,num_work_tours,"_PERSON_TOUR_COUNT('tour_type==%s' % _Q('work'), persons, tours).fillna(0)" +work_and_school_and_worker,work_and_school_and_worker,(persons.mandatory_tour_frequency == asim_enum.MandatoryTourFrequency.work_and_school) & persons.is_worker +work_and_school_and_student,work_and_school_and_student,(persons.mandatory_tour_frequency == asim_enum.MandatoryTourFrequency.work_and_school) & persons.is_student +,_MANDATORY,asim_enum.TourCategory.mandatory.value +,_WORK,asim_enum.TourPurpose.work.value +number of mandatory tours for each person,num_mand,"_PERSON_TOUR_COUNT('tour_category==%s' % _MANDATORY, persons, tours).fillna(0)" +number of work tours for each person,num_work_tours,"_PERSON_TOUR_COUNT('tour_type==%s' % _WORK, persons, tours).fillna(0)" diff --git a/activitysim/examples/prototype_mtc/configs/settings.yaml b/activitysim/examples/prototype_mtc/configs/settings.yaml index 6577b94b9..e54493744 100644 --- a/activitysim/examples/prototype_mtc/configs/settings.yaml +++ b/activitysim/examples/prototype_mtc/configs/settings.yaml @@ -193,30 +193,30 @@ models: - cdap_simulate - mandatory_tour_frequency - mandatory_tour_scheduling - - joint_tour_frequency - - joint_tour_composition - - joint_tour_participation - - joint_tour_destination - - joint_tour_scheduling - - non_mandatory_tour_frequency - - non_mandatory_tour_destination - - non_mandatory_tour_scheduling - - tour_mode_choice_simulate - - atwork_subtour_frequency - - atwork_subtour_destination - - atwork_subtour_scheduling - - atwork_subtour_mode_choice - - stop_frequency - - trip_purpose - - trip_destination - - trip_purpose_and_destination - - trip_scheduling - - trip_mode_choice - - write_data_dictionary - - track_skim_usage - - write_trip_matrices + # - joint_tour_frequency + # - joint_tour_composition + # - joint_tour_participation + # - joint_tour_destination + # - joint_tour_scheduling + # - non_mandatory_tour_frequency + # - non_mandatory_tour_destination + # - non_mandatory_tour_scheduling + # - tour_mode_choice_simulate + # - atwork_subtour_frequency + # - atwork_subtour_destination + # - atwork_subtour_scheduling + # - atwork_subtour_mode_choice + # - stop_frequency + # - trip_purpose + # - trip_destination + # - trip_purpose_and_destination + # - trip_scheduling + # - trip_mode_choice + # - write_data_dictionary + # - track_skim_usage + # - write_trip_matrices - write_tables - - summarize + # - summarize output_tables: h5_store: False diff --git a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_annotate_choosers_preprocessor.csv b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_annotate_choosers_preprocessor.csv index 3ef95955f..02eac5cb3 100644 --- a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_annotate_choosers_preprocessor.csv +++ b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_annotate_choosers_preprocessor.csv @@ -2,7 +2,7 @@ Description,Target,Expression #,, local,_DF_IS_TOUR,'tour_type' in df.columns ,number_of_participants,df.number_of_participants if _DF_IS_TOUR else 1 -,is_joint,(df.tour_category=='joint') if _DF_IS_TOUR else False +,is_joint,(df.tour_category==asim_enum.TourCategory.joint) if _DF_IS_TOUR else False #,, local,_HAVE_PARENT_TOURS,'parent_tour_id' in df.columns ,_parent_tour_mode,"reindex(tours.tour_mode, df.parent_tour_id) if _HAVE_PARENT_TOURS else ''" @@ -10,10 +10,10 @@ local,_DF_IS_TOUR,'tour_type' in df.columns ,work_tour_is_bike,_parent_tour_mode=='BIKE' ,work_tour_is_SOV,"_parent_tour_mode.isin(['DRIVEALONEFREE','DRIVEALONEPAY'])" #,, -,is_mandatory,(df.tour_category=='mandatory') if 'tour_category' in df.columns else False -,is_joint,(df.tour_category=='joint') if 'tour_category' in df.columns else False +,is_mandatory,(df.tour_category==asim_enum.TourCategory.mandatory) if 'tour_category' in df.columns else False +,is_joint,(df.tour_category==asim_enum.TourCategory.joint) if 'tour_category' in df.columns else False ,is_indiv,~is_joint -,is_atwork_subtour,(df.tour_category=='atwork') if 'tour_category' in df.columns else False +,is_atwork_subtour,(df.tour_category==asim_enum.TourCategory.atwork) if 'tour_category' in df.columns else False ,is_escort,(df.tour_type == 'escort') if _DF_IS_TOUR else False # FIXME why inverse of value of time? need better name?,, #,c_cost,(0.60 * c_ivt) / df.value_of_time @@ -58,7 +58,7 @@ local,_DF_IS_TOUR,'tour_type' in df.columns ,totalWaitSingleTNC,origSingleTNCWaitTime + destSingleTNCWaitTime ,totalWaitSharedTNC,origSharedTNCWaitTime + destSharedTNCWaitTime #,, -,_free_parking_available,(df.tour_type == 'work') & df.free_parking_at_work if _DF_IS_TOUR else False +,_free_parking_available,(df.tour_type == asim_enum.TourPurpose.work) & df.free_parking_at_work if _DF_IS_TOUR else False ,_dest_hourly_peak_parking_cost,"reindex(land_use.PRKCST, df[dest_col_name])" ,_dest_hourly_offpeak_parking_cost,"reindex(land_use.OPRKCST, df[dest_col_name])" ,_hourly_peak_parking_cost,"np.where(_free_parking_available, 0, _dest_hourly_peak_parking_cost)" diff --git a/activitysim/examples/prototype_mtc/test/configs/settings.yaml b/activitysim/examples/prototype_mtc/test/configs/settings.yaml index 7626237ca..21b0a3db8 100644 --- a/activitysim/examples/prototype_mtc/test/configs/settings.yaml +++ b/activitysim/examples/prototype_mtc/test/configs/settings.yaml @@ -23,5 +23,8 @@ output_tables: sort: True tables: - trips + - persons + - households + - tours recode_pipeline_columns: False diff --git a/activitysim/examples/prototype_mtc/test/test_mtc.py b/activitysim/examples/prototype_mtc/test/test_mtc.py index b72ac5c49..1a2b81d03 100644 --- a/activitysim/examples/prototype_mtc/test/test_mtc.py +++ b/activitysim/examples/prototype_mtc/test/test_mtc.py @@ -92,9 +92,11 @@ def regress(): else: subprocess.run([sys.executable, file_path] + run_args, check=True) - regress() + #regress() +import pytest +@pytest.mark.menow def test_mtc(): run_test_mtc(multiprocess=False)