From 44e3c21c414cf7b843ed79b3cc3c76cadbd078da Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 16 Aug 2024 17:33:51 -0700 Subject: [PATCH 01/11] multiprocess initial commit --- .../models/non_mandatory_tour_frequency.py | 10 +- activitysim/abm/models/school_escorting.py | 5 +- activitysim/abm/models/stop_frequency.py | 12 +- activitysim/core/estimation.py | 35 +++++- activitysim/core/steps/output.py | 113 ++++++++++++++++++ 5 files changed, 165 insertions(+), 10 deletions(-) diff --git a/activitysim/abm/models/non_mandatory_tour_frequency.py b/activitysim/abm/models/non_mandatory_tour_frequency.py index e0fb4817e..118967597 100644 --- a/activitysim/abm/models/non_mandatory_tour_frequency.py +++ b/activitysim/abm/models/non_mandatory_tour_frequency.py @@ -288,14 +288,18 @@ def non_mandatory_tour_frequency( ) if estimator: - estimator.write_spec(model_settings, bundle_directory=True) + bundle_directory = True + # writing to separte subdirectory for each segment if multiprocessing + if state.settings.multiprocess: + bundle_directory = False + estimator.write_spec(model_settings, bundle_directory=bundle_directory) estimator.write_model_settings( - model_settings, model_settings_file_name, bundle_directory=True + model_settings, model_settings_file_name, bundle_directory=bundle_directory ) # preserving coefficients file name makes bringing back updated coefficients more straightforward estimator.write_coefficients(coefficients_df, segment_settings) estimator.write_choosers(chooser_segment) - estimator.write_alternatives(alternatives, bundle_directory=True) + estimator.write_alternatives(alternatives, bundle_directory=bundle_directory) # FIXME #interaction_simulate_estimation_requires_chooser_id_in_df_column # shuold we do it here or have interaction_simulate do it? diff --git a/activitysim/abm/models/school_escorting.py b/activitysim/abm/models/school_escorting.py index 22fea9e52..d8305740a 100644 --- a/activitysim/abm/models/school_escorting.py +++ b/activitysim/abm/models/school_escorting.py @@ -493,7 +493,10 @@ def school_escorting( coefficients_df, file_name=stage.upper() + "_COEFFICIENTS" ) estimator.write_choosers(choosers) - estimator.write_alternatives(alts, bundle_directory=True) + if state.settings.multiprocess: + estimator.write_alternatives(alts, bundle_directory=False) + else: + estimator.write_alternatives(alts, bundle_directory=True) # FIXME #interaction_simulate_estimation_requires_chooser_id_in_df_column # shuold we do it here or have interaction_simulate do it? diff --git a/activitysim/abm/models/stop_frequency.py b/activitysim/abm/models/stop_frequency.py index 2f0253f21..70755ff86 100644 --- a/activitysim/abm/models/stop_frequency.py +++ b/activitysim/abm/models/stop_frequency.py @@ -197,9 +197,15 @@ def stop_frequency( if estimator: estimator.write_spec(segment_settings, bundle_directory=False) - estimator.write_model_settings( - model_settings, model_settings_file_name, bundle_directory=True - ) + # writing to separte subdirectory for each segment if multiprocessing + if state.settings.multiprocess: + estimator.write_model_settings( + model_settings, model_settings_file_name, bundle_directory=False + ) + else: + estimator.write_model_settings( + model_settings, model_settings_file_name, bundle_directory=True + ) estimator.write_coefficients(coefficients_df, segment_settings) estimator.write_choosers(chooser_segment) diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py index 6e02aca76..124b33bd5 100644 --- a/activitysim/core/estimation.py +++ b/activitysim/core/estimation.py @@ -19,7 +19,13 @@ ESTIMATION_SETTINGS_FILE_NAME = "estimation.yaml" -def unlink_files(directory_path, file_types=("csv", "yaml")): +def unlink_files(directory_path, file_types=("csv", "yaml", "parquet")): + """ + Deletes existing files in directory_path with file_types extensions. + """ + if not os.path.exists(directory_path): + return + for file_name in os.listdir(directory_path): if file_name.endswith(file_types): file_path = os.path.join(directory_path, file_name) @@ -31,6 +37,16 @@ def unlink_files(directory_path, file_types=("csv", "yaml")): print(e) +def estimation_enabled(state): + """ + Returns True if estimation.yaml exists in the configs directory. + """ + settings = state.filesystem.read_model_settings( + ESTIMATION_SETTINGS_FILE_NAME, mandatory=False + ) + return settings is not None + + class Estimator: def __init__( self, state: workflow.State, bundle_name, model_name, estimation_table_recipes @@ -50,12 +66,12 @@ def __init__( os.makedirs(output_dir) # make directory if needed # delete estimation files - unlink_files(self.output_directory(), file_types=("csv", "yaml")) + unlink_files(self.output_directory(), file_types=("csv", "yaml", "parquet")) if self.bundle_name != self.model_name: # kind of inelegant to always delete these, but ok as they are redundantly recreated for each sub model unlink_files( self.output_directory(bundle_directory=True), - file_types=("csv", "yaml"), + file_types=("csv", "yaml", "parquet"), ) # FIXME - not required? @@ -139,6 +155,9 @@ def output_directory(self, bundle_directory=False): if self.bundle_name != self.model_name and not bundle_directory: dir = os.path.join(dir, self.model_name) + if self.state.settings.multiprocess: + dir = os.path.join(dir, self.state.get_injectable("pipeline_file_prefix")) + return dir def output_file_path(self, table_name, file_type=None, bundle_directory=False): @@ -546,6 +565,16 @@ def initialize_settings(self, state): ), "Index col '%s' not in survey_table '%s' in file: %s % (index_col, table_name, file_path)" df.set_index(index_col, inplace=True) + # if multiprocessing then only return the households that are in the pipeline + if state.settings.multiprocess: + pipeline_hh_ids = state.get_table("households").index + if table_name == "households": + df = df[df.index.isin(pipeline_hh_ids)] + assert pipeline_hh_ids.equals(df.index), "household_ids not equal between survey and pipeline" + else: + assert "household_id" in df.columns + df = df[df.household_id.isin(pipeline_hh_ids)] + # add the table df to survey_tables table_info["df"] = df diff --git a/activitysim/core/steps/output.py b/activitysim/core/steps/output.py index b9d7cc13d..6116fda61 100644 --- a/activitysim/core/steps/output.py +++ b/activitysim/core/steps/output.py @@ -4,6 +4,9 @@ import logging import sys +import os +import shutil +from pathlib import Path import numpy as np import pandas as pd @@ -13,6 +16,7 @@ from activitysim.core import configuration, workflow from activitysim.core.workflow.checkpoint import CHECKPOINT_NAME +from activitysim.core.estimation import estimation_enabled logger = logging.getLogger(__name__) @@ -224,6 +228,111 @@ def write_data_dictionary(state: workflow.State) -> None: print(f"{info}\n", file=output_file) +def find_lowest_level_directories(starting_directory): + lowest_dirs = list() + + for root, dirs, files in os.walk(starting_directory): + if not dirs: + lowest_dirs.append(root) + + return lowest_dirs + + +def concat_and_write_edb(df_concat_dict, write_dir): + # concatenate the dataframes and output final file + for table_name, df_array in df_concat_dict.items(): + df = pd.concat(df_array) + if table_name.endswith(".csv"): + df.to_csv(os.path.join(write_dir, table_name), index=False) + elif table_name.endswith(".parquet"): + df.to_parquet(os.path.join(write_dir, table_name), index=True) + else: + raise ValueError(f"Unknown file type {table_name}") + + +def coalesce_estimation_data_bundles(state): + """ + In estimation mode, estimation data bundles are written to separate subdirectories for each subprocess. + This model will go through each subdirectory and move the files to the parent directory. + This will only occur if the lowest level directory contains the multiprocess step names. + """ + + logger.info("Coalescing Estimation Data Bundles") + + edb_dir = state.filesystem.get_output_dir("estimation_data_bundle") + + lowest_dirs = find_lowest_level_directories(edb_dir) + + multiprocessing_step_names = [step.name for step in state.settings.multiprocess_steps] + lowest_dirs = [dir for dir in lowest_dirs if any(step in dir for step in multiprocessing_step_names)] + + if len(lowest_dirs) == 0: + logger.info("No estimation data bundles to coalesce") + return + + prev_edb = None + df_concat_dict = {} + + # loop through each lowest level directory + for dir in lowest_dirs: + logger.debug(f"Coalescing {dir}") + # get the parent directory + cur_edb = Path(dir).parent.absolute() + + # check if we have moved onto a new EDB + is_same_edb = (cur_edb == prev_edb) + + for file in os.listdir(dir): + # get the file path + file_path = os.path.join(dir, file) + + # look for files that are duplicated across subprocesses + is_coefs_file = file.endswith(".csv") and "coef" in file + is_settings_file = file.endswith(".yaml") + is_spec_file = file.endswith(".csv") and "SPEC" in file + is_landuse_file = file.endswith("_landuse.csv") + is_size_terms_file = file.endswith("_size_terms.csv") + is_duplicate_file = ( + is_coefs_file + or is_spec_file + or is_settings_file + or is_landuse_file + or is_size_terms_file + ) + + if is_duplicate_file and not is_same_edb: + # copy the file to the parent directory + shutil.copy(file_path, os.path.join(cur_edb, file)) + + if (not is_same_edb) and (len(df_concat_dict) > 0) and (len(df_concat_dict[list(df_concat_dict.keys())[0]]) > 1): + concat_and_write_edb(df_concat_dict, cur_edb) + + # reset edb dir and dictionary + logger.info(f"Coalesced {cur_edb}") + prev_edb = cur_edb + df_concat_dict = {} + + if not is_duplicate_file: + # read file and store in dictionary + if file.endswith(".csv"): + df = pd.read_csv(file_path, low_memory=False) + elif file.endswith(".parquet"): + df = pd.read_parquet(file_path) + + if file in df_concat_dict.keys(): + df_concat_dict[file].append(df) + else: + df_concat_dict[file] = [df] + + # delete the directory now that we have gone through all the files + # os.rmdir(dir) + + # need to concatenate the last set of dataframes + concat_and_write_edb(df_concat_dict, cur_edb) + + return + + @workflow.step def write_tables(state: workflow.State) -> None: """ @@ -434,3 +543,7 @@ def map_func(x): parquet.write_table(dt, file_path) else: raise ValueError(f"unknown file_type {file_type}") + + is_estimation = estimation_enabled(state) + if state.settings.multiprocess and is_estimation: + coalesce_estimation_data_bundles(state) From 9b29350b85e3f914d441a35ad946469c44ec1f2e Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 16 Aug 2024 17:41:41 -0700 Subject: [PATCH 02/11] blacken --- .../models/non_mandatory_tour_frequency.py | 8 ++++-- activitysim/core/estimation.py | 4 ++- activitysim/core/steps/output.py | 26 +++++++++++++------ 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/activitysim/abm/models/non_mandatory_tour_frequency.py b/activitysim/abm/models/non_mandatory_tour_frequency.py index 118967597..4d80c26ba 100644 --- a/activitysim/abm/models/non_mandatory_tour_frequency.py +++ b/activitysim/abm/models/non_mandatory_tour_frequency.py @@ -294,12 +294,16 @@ def non_mandatory_tour_frequency( bundle_directory = False estimator.write_spec(model_settings, bundle_directory=bundle_directory) estimator.write_model_settings( - model_settings, model_settings_file_name, bundle_directory=bundle_directory + model_settings, + model_settings_file_name, + bundle_directory=bundle_directory, ) # preserving coefficients file name makes bringing back updated coefficients more straightforward estimator.write_coefficients(coefficients_df, segment_settings) estimator.write_choosers(chooser_segment) - estimator.write_alternatives(alternatives, bundle_directory=bundle_directory) + estimator.write_alternatives( + alternatives, bundle_directory=bundle_directory + ) # FIXME #interaction_simulate_estimation_requires_chooser_id_in_df_column # shuold we do it here or have interaction_simulate do it? diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py index 124b33bd5..0de69b3d9 100644 --- a/activitysim/core/estimation.py +++ b/activitysim/core/estimation.py @@ -570,7 +570,9 @@ def initialize_settings(self, state): pipeline_hh_ids = state.get_table("households").index if table_name == "households": df = df[df.index.isin(pipeline_hh_ids)] - assert pipeline_hh_ids.equals(df.index), "household_ids not equal between survey and pipeline" + assert pipeline_hh_ids.equals( + df.index + ), "household_ids not equal between survey and pipeline" else: assert "household_id" in df.columns df = df[df.household_id.isin(pipeline_hh_ids)] diff --git a/activitysim/core/steps/output.py b/activitysim/core/steps/output.py index 6116fda61..39e2d9650 100644 --- a/activitysim/core/steps/output.py +++ b/activitysim/core/steps/output.py @@ -263,13 +263,19 @@ def coalesce_estimation_data_bundles(state): lowest_dirs = find_lowest_level_directories(edb_dir) - multiprocessing_step_names = [step.name for step in state.settings.multiprocess_steps] - lowest_dirs = [dir for dir in lowest_dirs if any(step in dir for step in multiprocessing_step_names)] + multiprocessing_step_names = [ + step.name for step in state.settings.multiprocess_steps + ] + lowest_dirs = [ + dir + for dir in lowest_dirs + if any(step in dir for step in multiprocessing_step_names) + ] if len(lowest_dirs) == 0: logger.info("No estimation data bundles to coalesce") return - + prev_edb = None df_concat_dict = {} @@ -280,7 +286,7 @@ def coalesce_estimation_data_bundles(state): cur_edb = Path(dir).parent.absolute() # check if we have moved onto a new EDB - is_same_edb = (cur_edb == prev_edb) + is_same_edb = cur_edb == prev_edb for file in os.listdir(dir): # get the file path @@ -293,8 +299,8 @@ def coalesce_estimation_data_bundles(state): is_landuse_file = file.endswith("_landuse.csv") is_size_terms_file = file.endswith("_size_terms.csv") is_duplicate_file = ( - is_coefs_file - or is_spec_file + is_coefs_file + or is_spec_file or is_settings_file or is_landuse_file or is_size_terms_file @@ -304,7 +310,11 @@ def coalesce_estimation_data_bundles(state): # copy the file to the parent directory shutil.copy(file_path, os.path.join(cur_edb, file)) - if (not is_same_edb) and (len(df_concat_dict) > 0) and (len(df_concat_dict[list(df_concat_dict.keys())[0]]) > 1): + if ( + (not is_same_edb) + and (len(df_concat_dict) > 0) + and (len(df_concat_dict[list(df_concat_dict.keys())[0]]) > 1) + ): concat_and_write_edb(df_concat_dict, cur_edb) # reset edb dir and dictionary @@ -543,7 +553,7 @@ def map_func(x): parquet.write_table(dt, file_path) else: raise ValueError(f"unknown file_type {file_type}") - + is_estimation = estimation_enabled(state) if state.settings.multiprocess and is_estimation: coalesce_estimation_data_bundles(state) From 3434c9540be1cea8f1c2623c049441ef6809b2d4 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 6 Sep 2024 09:18:32 -0700 Subject: [PATCH 03/11] parquet format for EDBs --- activitysim/core/estimation.py | 162 +++++++++++++++++++++++-------- activitysim/core/steps/output.py | 3 +- 2 files changed, 124 insertions(+), 41 deletions(-) diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py index 0de69b3d9..5b815183e 100644 --- a/activitysim/core/estimation.py +++ b/activitysim/core/estimation.py @@ -5,6 +5,7 @@ import logging import os import shutil +from pathlib import Path import pandas as pd import yaml @@ -49,7 +50,12 @@ def estimation_enabled(state): class Estimator: def __init__( - self, state: workflow.State, bundle_name, model_name, estimation_table_recipes + self, + state: workflow.State, + bundle_name, + model_name, + estimation_table_recipes, + settings, ): logger.info("Initialize Estimator for'%s'" % (model_name,)) @@ -59,6 +65,7 @@ def __init__( self.settings_name = model_name self.estimation_table_recipes = estimation_table_recipes self.estimating = True + self.settings = settings # ensure the output data directory exists output_dir = self.output_directory() @@ -179,8 +186,43 @@ def output_file_path(self, table_name, file_type=None, bundle_directory=False): return os.path.join(output_dir, file_name) + def write_parquet(self, df, file_path, append=False): + """Convert DF to be parquet compliant and write to disk""" + # Ensure column names are strings for parquet + df.columns = df.columns.astype(str) + + assert (not os.path.isfile(file_path)) or ( + append == True + ), f"file already exists: {file_path}" + + # Explicitly set the data types of the columns + for col in df.columns: + if df[col].dtype == "object": + df[col] = df[col].astype(str) + elif df[col].dtype == "int": + df[col] = df[col].astype("int64") + elif df[col].dtype == "float": + df[col] = df[col].astype("float64") + elif df[col].dtype == "float16": # Handle halffloat type + df[col] = df[col].astype("float32") + else: + # Convert any other unsupported types to string + df[col] = df[col].astype(str) + + self.debug(f"writing table: {file_path}") + if append and os.path.isfile(file_path): + df.to_parquet(file_path, engine="fastparquet", append=True) + else: + df.to_parquet(file_path) + def write_table( - self, df, table_name, index=True, append=True, bundle_directory=False + self, + df, + table_name, + index=True, + append=True, + bundle_directory=False, + filetype="csv", ): """ @@ -192,6 +234,8 @@ def write_table( index: booelan append: boolean bundle_directory: boolean + filetype: str + csv or parquet """ @@ -205,21 +249,23 @@ def cache_table(df, table_name, append): else: self.tables[table_name] = df.copy() - def write_table(df, table_name, index, append, bundle_directory): - if table_name.endswith(".csv"): - # pass through filename without adding model or bundle name prefix - file_path = os.path.join( - self.output_directory(bundle_directory), table_name - ) - else: - file_path = self.output_file_path(table_name, "csv", bundle_directory) + def write_table(df, table_name, index, append, bundle_directory, filetype): + # remove file extension if present + table_name = Path(table_name).stem + # set new full file path with desired file type + file_path = self.output_file_path(table_name, filetype, bundle_directory) + + # check if file exists file_exists = os.path.isfile(file_path) if file_exists and not append: raise RuntimeError( "write_table %s append=False and file exists: %s" % (table_name, file_path) ) - df.to_csv(file_path, mode="a", index=index, header=(not file_exists)) + if filetype == "csv": + df.to_csv(file_path, mode="a", index=index, header=(not file_exists)) + else: + self.write_parquet(df, file_path, append) assert self.estimating @@ -232,18 +278,14 @@ def write_table(df, table_name, index, append, bundle_directory): self.debug("write_table cache: %s" % table_name) if write: - write_table(df, table_name, index, append, bundle_directory) + write_table(df, table_name, index, append, bundle_directory, filetype) self.debug("write_table write: %s" % table_name) def write_omnibus_table(self): if len(self.omnibus_tables) == 0: return - settings = self.state.filesystem.read_model_settings( - ESTIMATION_SETTINGS_FILE_NAME, mandatory=False - ) - - edbs_to_skip = settings.get("SKIP_BUNDLE_WRITE_FOR", []) + edbs_to_skip = self.settings.get("SKIP_BUNDLE_WRITE_FOR", []) if self.bundle_name in edbs_to_skip: self.debug(f"Skipping write to disk for {self.bundle_name}") return @@ -274,13 +316,20 @@ def write_omnibus_table(self): self.debug(f"sorting tables: {table_names}") df.sort_index(ascending=True, inplace=True, kind="mergesort") - file_path = self.output_file_path(omnibus_table, "csv") - assert not os.path.isfile(file_path) + filetype = self.settings.get("EDB_FILETYPE", "csv") + assert filetype in ["csv", "parquet"] - self.debug(f"writing table: {file_path}") - df.to_csv(file_path, mode="a", index=True, header=True) + if filetype == "csv": + file_path = self.output_file_path(omnibus_table, "csv") + assert not os.path.isfile(file_path) + + self.debug(f"writing table: {file_path}") + df.to_csv(file_path, mode="a", index=True, header=True) + else: + file_path = self.output_file_path(omnibus_table, "parquet") + self.write_parquet(df, file_path, append=False) - self.debug("write_omnibus_choosers: %s" % file_path) + self.debug("wrote_omnibus_choosers: %s" % file_path) def write_dict(self, d, dict_name, bundle_directory): assert self.estimating @@ -323,7 +372,7 @@ def write_coefficients( base_file_name = os.path.basename(file_name) assert self.estimating - self.write_table(coefficients_df, base_file_name, append=False) + self.write_table(coefficients_df, base_file_name, append=False, filetype="csv") def write_coefficients_template(self, model_settings): assert self.estimating @@ -334,22 +383,37 @@ def write_coefficients_template(self, model_settings): self.state.filesystem, model_settings ) tag = "coefficients_template" - self.write_table(coefficients_df, tag, append=False) + self.write_table(coefficients_df, tag, append=False, filetype="csv") def write_choosers(self, choosers_df): - self.write_table(choosers_df, "choosers", append=True) + self.write_table( + choosers_df, + "choosers", + append=True, + filetype=self.settings.get("EDB_FILETYPE", "csv"), + ) def write_choices(self, choices): if isinstance(choices, pd.Series): choices = choices.to_frame(name="model_choice") assert list(choices.columns) == ["model_choice"] - self.write_table(choices, "choices", append=True) + self.write_table( + choices, + "choices", + append=True, + filetype=self.settings.get("EDB_FILETYPE", "csv"), + ) def write_override_choices(self, choices): if isinstance(choices, pd.Series): choices = choices.to_frame(name="override_choice") assert list(choices.columns) == ["override_choice"] - self.write_table(choices, "override_choices", append=True) + self.write_table( + choices, + "override_choices", + append=True, + filetype=self.settings.get("EDB_FILETYPE", "csv"), + ) def write_constants(self, constants): self.write_dict(self, constants, "model_constants") @@ -463,10 +527,20 @@ def melt_alternatives(self, df): def write_interaction_expression_values(self, df): df = self.melt_alternatives(df) - self.write_table(df, "interaction_expression_values", append=True) + self.write_table( + df, + "interaction_expression_values", + append=True, + filetype=self.settings.get("EDB_FILETYPE", "csv"), + ) def write_expression_values(self, df): - self.write_table(df, "expression_values", append=True) + self.write_table( + df, + "expression_values", + append=True, + filetype=self.settings.get("EDB_FILETYPE", "csv"), + ) def write_alternatives(self, alternatives_df, bundle_directory=False): self.write_table( @@ -479,13 +553,19 @@ def write_alternatives(self, alternatives_df, bundle_directory=False): def write_interaction_sample_alternatives(self, alternatives_df): alternatives_df = self.melt_alternatives(alternatives_df) self.write_table( - alternatives_df, "interaction_sample_alternatives", append=True + alternatives_df, + "interaction_sample_alternatives", + append=True, + filetype=self.settings.get("EDB_FILETYPE", "csv"), ) def write_interaction_simulate_alternatives(self, interaction_df): interaction_df = self.melt_alternatives(interaction_df) self.write_table( - interaction_df, "interaction_simulate_alternatives", append=True + interaction_df, + "interaction_simulate_alternatives", + append=True, + filetype=self.settings.get("EDB_FILETYPE", "csv"), ) def get_survey_values(self, model_values, table_name, column_names): @@ -520,6 +600,7 @@ def __init__(self): self.estimation_table_recipes = {} self.model_estimation_table_types = {} self.estimating = {} + self.settings = None def initialize_settings(self, state): # FIXME - can't we just initialize in init and handle no-presence of settings file as not enabled @@ -527,23 +608,25 @@ def initialize_settings(self, state): return assert not self.settings_initialized - settings = state.filesystem.read_model_settings( + self.settings = state.filesystem.read_model_settings( ESTIMATION_SETTINGS_FILE_NAME, mandatory=False ) - if not settings: - # if the model settings file is not found, we are not in estimation mode. + if not self.settings: + # if the model self.settings file is not found, we are not in estimation mode. self.enabled = False else: - self.enabled = settings.get("enable", "True") - self.bundles = settings.get("bundles", []) + self.enabled = self.settings.get("enable", "True") + self.bundles = self.settings.get("bundles", []) - self.model_estimation_table_types = settings.get( + self.model_estimation_table_types = self.settings.get( "model_estimation_table_types", {} ) - self.estimation_table_recipes = settings.get("estimation_table_recipes", {}) + self.estimation_table_recipes = self.settings.get( + "estimation_table_recipes", {} + ) if self.enabled: - self.survey_tables = settings.get("survey_tables", {}) + self.survey_tables = self.settings.get("survey_tables", {}) for table_name, table_info in self.survey_tables.items(): assert ( "file_name" in table_info @@ -643,6 +726,7 @@ def begin_estimation( estimation_table_recipes=self.estimation_table_recipes[ model_estimation_table_type ], + settings=self.settings, ) return self.estimating[model_name] diff --git a/activitysim/core/steps/output.py b/activitysim/core/steps/output.py index 39e2d9650..0306fcff4 100644 --- a/activitysim/core/steps/output.py +++ b/activitysim/core/steps/output.py @@ -318,7 +318,6 @@ def coalesce_estimation_data_bundles(state): concat_and_write_edb(df_concat_dict, cur_edb) # reset edb dir and dictionary - logger.info(f"Coalesced {cur_edb}") prev_edb = cur_edb df_concat_dict = {} @@ -335,7 +334,7 @@ def coalesce_estimation_data_bundles(state): df_concat_dict[file] = [df] # delete the directory now that we have gone through all the files - # os.rmdir(dir) + shutil.rmtree(dir) # need to concatenate the last set of dataframes concat_and_write_edb(df_concat_dict, cur_edb) From 914b9ca7109cf478c29c2147df19baa2b6cfae02 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 20 Sep 2024 11:59:24 -0700 Subject: [PATCH 04/11] adding pkl, fixing edb concat and write --- activitysim/core/estimation.py | 35 ++++++++++++++++++------ activitysim/core/steps/output.py | 47 ++++++++++++++++++++++---------- 2 files changed, 60 insertions(+), 22 deletions(-) diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py index 5b815183e..24b5f3e53 100644 --- a/activitysim/core/estimation.py +++ b/activitysim/core/estimation.py @@ -186,7 +186,7 @@ def output_file_path(self, table_name, file_type=None, bundle_directory=False): return os.path.join(output_dir, file_name) - def write_parquet(self, df, file_path, append=False): + def write_parquet(self, df, file_path, index, append=False): """Convert DF to be parquet compliant and write to disk""" # Ensure column names are strings for parquet df.columns = df.columns.astype(str) @@ -211,9 +211,9 @@ def write_parquet(self, df, file_path, append=False): self.debug(f"writing table: {file_path}") if append and os.path.isfile(file_path): - df.to_parquet(file_path, engine="fastparquet", append=True) + df.to_parquet(file_path, engine="fastparquet", append=True, index=index) else: - df.to_parquet(file_path) + df.to_parquet(file_path, index=index) def write_table( self, @@ -235,7 +235,7 @@ def write_table( append: boolean bundle_directory: boolean filetype: str - csv or parquet + csv or parquet or pkl """ @@ -264,8 +264,20 @@ def write_table(df, table_name, index, append, bundle_directory, filetype): ) if filetype == "csv": df.to_csv(file_path, mode="a", index=index, header=(not file_exists)) + elif filetype == "parquet": + self.write_parquet(df, file_path, index, append) + elif filetype == "pkl": + if append: + # read the previous df and concat + prev_df = pd.read_pickle(file_path) + df = pd.concat([prev_df, df]) + if index == False: + df.reset_index(drop=True, inplace=True) + df.to_pickle(file_path) else: - self.write_parquet(df, file_path, append) + raise RuntimeError( + f"Unsupported filetype: {filetype}, allowed options are csv, parquet, pkl" + ) assert self.estimating @@ -317,7 +329,6 @@ def write_omnibus_table(self): df.sort_index(ascending=True, inplace=True, kind="mergesort") filetype = self.settings.get("EDB_FILETYPE", "csv") - assert filetype in ["csv", "parquet"] if filetype == "csv": file_path = self.output_file_path(omnibus_table, "csv") @@ -325,9 +336,17 @@ def write_omnibus_table(self): self.debug(f"writing table: {file_path}") df.to_csv(file_path, mode="a", index=True, header=True) - else: + + elif filetype == "parquet": file_path = self.output_file_path(omnibus_table, "parquet") - self.write_parquet(df, file_path, append=False) + self.write_parquet(df, file_path, index=True, append=False) + + elif filetype == "pkl": + file_path = self.output_file_path(omnibus_table, "pkl") + df.to_pickle(file_path) + + else: + raise RuntimeError(f"Unsupported filetype: {filetype}") self.debug("wrote_omnibus_choosers: %s" % file_path) diff --git a/activitysim/core/steps/output.py b/activitysim/core/steps/output.py index 0306fcff4..d4ebccdb3 100644 --- a/activitysim/core/steps/output.py +++ b/activitysim/core/steps/output.py @@ -242,10 +242,19 @@ def concat_and_write_edb(df_concat_dict, write_dir): # concatenate the dataframes and output final file for table_name, df_array in df_concat_dict.items(): df = pd.concat(df_array) + + # sort the dataframe by index + if df.index.name is not None: + df = df.sort_index() + else: + df = df.sort_values(by=df.columns[0]) + if table_name.endswith(".csv"): df.to_csv(os.path.join(write_dir, table_name), index=False) elif table_name.endswith(".parquet"): df.to_parquet(os.path.join(write_dir, table_name), index=True) + elif table_name.endswith(".pkl"): + df.to_pickle(os.path.join(write_dir, table_name)) else: raise ValueError(f"Unknown file type {table_name}") @@ -255,6 +264,7 @@ def coalesce_estimation_data_bundles(state): In estimation mode, estimation data bundles are written to separate subdirectories for each subprocess. This model will go through each subdirectory and move the files to the parent directory. This will only occur if the lowest level directory contains the multiprocess step names. + Only multiprocess step names are used because that's how EDBs are written in estimation mode. """ logger.info("Coalescing Estimation Data Bundles") @@ -284,18 +294,32 @@ def coalesce_estimation_data_bundles(state): logger.debug(f"Coalescing {dir}") # get the parent directory cur_edb = Path(dir).parent.absolute() + if prev_edb is None: + prev_edb = cur_edb # check if we have moved onto a new EDB is_same_edb = cur_edb == prev_edb - for file in os.listdir(dir): + # if we have moved onto a new EDB, concatenate the dataframes and write the final files + if ( + (not is_same_edb) + and (len(df_concat_dict) > 0) + and (len(df_concat_dict[list(df_concat_dict.keys())[0]]) > 1) + ): + concat_and_write_edb(df_concat_dict, prev_edb) + + # reset edb dir and dictionary + prev_edb = cur_edb + df_concat_dict = {} + + for i, file in enumerate(os.listdir(dir)): # get the file path file_path = os.path.join(dir, file) # look for files that are duplicated across subprocesses is_coefs_file = file.endswith(".csv") and "coef" in file is_settings_file = file.endswith(".yaml") - is_spec_file = file.endswith(".csv") and "SPEC" in file + is_spec_file = file.endswith(".csv") and ("spec" in file.lower()) is_landuse_file = file.endswith("_landuse.csv") is_size_terms_file = file.endswith("_size_terms.csv") is_duplicate_file = ( @@ -310,23 +334,18 @@ def coalesce_estimation_data_bundles(state): # copy the file to the parent directory shutil.copy(file_path, os.path.join(cur_edb, file)) - if ( - (not is_same_edb) - and (len(df_concat_dict) > 0) - and (len(df_concat_dict[list(df_concat_dict.keys())[0]]) > 1) - ): - concat_and_write_edb(df_concat_dict, cur_edb) - - # reset edb dir and dictionary - prev_edb = cur_edb - df_concat_dict = {} - if not is_duplicate_file: # read file and store in dictionary if file.endswith(".csv"): df = pd.read_csv(file_path, low_memory=False) elif file.endswith(".parquet"): df = pd.read_parquet(file_path) + elif file.endswith(".pkl"): + df = pd.read_pickle(file_path) + else: + raise ValueError( + f"Unknown file type found {file}, expect csv, parquet, or pkl" + ) if file in df_concat_dict.keys(): df_concat_dict[file].append(df) @@ -334,7 +353,7 @@ def coalesce_estimation_data_bundles(state): df_concat_dict[file] = [df] # delete the directory now that we have gone through all the files - shutil.rmtree(dir) + # shutil.rmtree(dir) # need to concatenate the last set of dataframes concat_and_write_edb(df_concat_dict, cur_edb) From d2e181f089a86f1db057db307bdbcb6ac0ad4989 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 23 Sep 2024 13:01:51 -0700 Subject: [PATCH 05/11] fixing double naming of coefficient files --- activitysim/core/estimation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py index 24b5f3e53..1fb21522e 100644 --- a/activitysim/core/estimation.py +++ b/activitysim/core/estimation.py @@ -176,7 +176,9 @@ def output_file_path(self, table_name, file_type=None, bundle_directory=False): if bundle_directory: file_name = f"{self.bundle_name}_{table_name}" else: - if self.model_name == self.bundle_name: + if '_coefficients' in table_name: + file_name = f"{table_name}" + elif self.model_name == self.bundle_name: file_name = f"{self.model_name}_{table_name}" else: file_name = f"{self.bundle_name}_{table_name}" From c138f0f1578e9111423a5e5aa451c3c150a7da63 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 23 Sep 2024 13:15:50 -0700 Subject: [PATCH 06/11] blacken --- activitysim/core/estimation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py index 1fb21522e..aedb84510 100644 --- a/activitysim/core/estimation.py +++ b/activitysim/core/estimation.py @@ -176,7 +176,7 @@ def output_file_path(self, table_name, file_type=None, bundle_directory=False): if bundle_directory: file_name = f"{self.bundle_name}_{table_name}" else: - if '_coefficients' in table_name: + if "_coefficients" in table_name: file_name = f"{table_name}" elif self.model_name == self.bundle_name: file_name = f"{self.model_name}_{table_name}" From 6d35f9fcfbf9193e9d5d751653fe6610130cbe9b Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 23 Sep 2024 16:03:11 -0700 Subject: [PATCH 07/11] fixing missing cdap coefficients file, write pickle function --- activitysim/abm/models/cdap.py | 2 +- activitysim/core/estimation.py | 58 ++++++++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/activitysim/abm/models/cdap.py b/activitysim/abm/models/cdap.py index 6776c06c7..2b9a105a5 100644 --- a/activitysim/abm/models/cdap.py +++ b/activitysim/abm/models/cdap.py @@ -180,7 +180,7 @@ def cdap_simulate( estimator.write_coefficients(coefficients_df, model_settings) estimator.write_table( cdap_interaction_coefficients, - "interaction_coefficients", + "cdap_interaction_coefficients", index=False, append=False, ) diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py index aedb84510..6c88dcccd 100644 --- a/activitysim/core/estimation.py +++ b/activitysim/core/estimation.py @@ -20,7 +20,7 @@ ESTIMATION_SETTINGS_FILE_NAME = "estimation.yaml" -def unlink_files(directory_path, file_types=("csv", "yaml", "parquet")): +def unlink_files(directory_path, file_types=("csv", "yaml", "parquet", "pkl")): """ Deletes existing files in directory_path with file_types extensions. """ @@ -33,9 +33,9 @@ def unlink_files(directory_path, file_types=("csv", "yaml", "parquet")): try: if os.path.isfile(file_path): os.unlink(file_path) - print(f"deleted {file_path}") + logger.debug(f"deleted {file_path}") except Exception as e: - print(e) + logger.error(e) def estimation_enabled(state): @@ -212,10 +212,40 @@ def write_parquet(self, df, file_path, index, append=False): df[col] = df[col].astype(str) self.debug(f"writing table: {file_path}") + # want parquet file to be exactly the same as df read from csv + # therefore we are resetting the index into a column if we want to keep it + # if we don't want to keep it, we are dropping it on write with index=False + if index: + df = df.reset_index(drop=False) + if append and os.path.isfile(file_path): - df.to_parquet(file_path, engine="fastparquet", append=True, index=index) + df.to_parquet(file_path, engine="fastparquet", append=True, index=False) + else: + df.to_parquet(file_path, index=False) + + def write_pickle(self, df, file_path, index, append=False): + """Write DF to disk as pickle""" + file_path = file_path.replace(".csv", ".pkl").replace(".parquet", ".pkl") + assert file_path.endswith(".pkl") + + # want pickle file to be exactly the same as df read from csv + # therefore we are resetting the index into a column if we want to keep it + # if we don't want to keep it, we are dropping it on write with index=False + if index: + df = df.reset_index(drop=False) else: - df.to_parquet(file_path, index=index) + df = df.reset_index(drop=True) + + assert (not os.path.isfile(file_path)) or ( + append == True + ), f"file already exists: {file_path}" + + if append: + # read the previous df and concat + prev_df = pd.read_pickle(file_path) + df = pd.concat([prev_df, df]) + + df.to_pickle(file_path) def write_table( self, @@ -267,15 +297,15 @@ def write_table(df, table_name, index, append, bundle_directory, filetype): if filetype == "csv": df.to_csv(file_path, mode="a", index=index, header=(not file_exists)) elif filetype == "parquet": - self.write_parquet(df, file_path, index, append) + try: + self.write_parquet(df, file_path, index, append) + except Exception as e: + logger.error( + f"Error writing parquet: {file_path} because {e}, falling back to pickle" + ) + self.write_pickle(df, file_path, index, append) elif filetype == "pkl": - if append: - # read the previous df and concat - prev_df = pd.read_pickle(file_path) - df = pd.concat([prev_df, df]) - if index == False: - df.reset_index(drop=True, inplace=True) - df.to_pickle(file_path) + self.write_pickle(df, file_path, index, append) else: raise RuntimeError( f"Unsupported filetype: {filetype}, allowed options are csv, parquet, pkl" @@ -345,7 +375,7 @@ def write_omnibus_table(self): elif filetype == "pkl": file_path = self.output_file_path(omnibus_table, "pkl") - df.to_pickle(file_path) + self.write_pickle(df, file_path, index=True, append=False) else: raise RuntimeError(f"Unsupported filetype: {filetype}") From 27c4ce48adc780ad024149096e4604c1bd09eb3b Mon Sep 17 00:00:00 2001 From: David Hensle Date: Tue, 24 Sep 2024 16:51:51 -0700 Subject: [PATCH 08/11] combact edb writing, index duplication, parquet datatypes --- activitysim/core/estimation.py | 43 ++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py index 6c88dcccd..793f1e532 100644 --- a/activitysim/core/estimation.py +++ b/activitysim/core/estimation.py @@ -199,14 +199,22 @@ def write_parquet(self, df, file_path, index, append=False): # Explicitly set the data types of the columns for col in df.columns: - if df[col].dtype == "object": - df[col] = df[col].astype(str) - elif df[col].dtype == "int": - df[col] = df[col].astype("int64") - elif df[col].dtype == "float": - df[col] = df[col].astype("float64") - elif df[col].dtype == "float16": # Handle halffloat type + if "int" in str(df[col].dtype): + pass + elif ( + df[col].dtype == "float16" + ): # Handle halffloat type not allowed in parquet df[col] = df[col].astype("float32") + elif "float" in str(df[col].dtype): + pass + elif df[col].dtype == "bool": + pass + elif df[col].dtype == "object": + # first try converting to numeric, if that fails, convert to string + try: + df[col] = pd.to_numeric(df[col], errors="raise") + except ValueError: + df[col] = df[col].astype(str) else: # Convert any other unsupported types to string df[col] = df[col].astype(str) @@ -216,6 +224,9 @@ def write_parquet(self, df, file_path, index, append=False): # therefore we are resetting the index into a column if we want to keep it # if we don't want to keep it, we are dropping it on write with index=False if index: + if df.index.name in df.columns: + # replace old index with new one + df.drop(columns=[df.index.name], inplace=True) df = df.reset_index(drop=False) if append and os.path.isfile(file_path): @@ -295,6 +306,9 @@ def write_table(df, table_name, index, append, bundle_directory, filetype): % (table_name, file_path) ) if filetype == "csv": + # check if index is in columns and drop it if so + if index and (df.index.name in df.columns): + df.drop(columns=df.index.name, inplace=True) df.to_csv(file_path, mode="a", index=index, header=(not file_exists)) elif filetype == "parquet": try: @@ -367,6 +381,9 @@ def write_omnibus_table(self): assert not os.path.isfile(file_path) self.debug(f"writing table: {file_path}") + # check if index is in columns and drop it if so + if df.index.name in df.columns: + df.drop(columns=df.index.name, inplace=True) df.to_csv(file_path, mode="a", index=True, header=True) elif filetype == "parquet": @@ -563,6 +580,18 @@ def melt_alternatives(self, df): # 31153,2,util_dist_0_1,1.0 # 31153,3,util_dist_0_1,1.0 + output_format = self.settings.get("EDB_ALTS_FILE_FORMAT", "verbose") + assert output_format in ["verbose", "compact"] + + if output_format == "compact": + # renumber the alt_id column to just count from 1 to n + # this loses the alt_id information, but drops all of the empty columns + # (can still get empty columns if not every chooser has same number of alts) + # (this can happen if the pick count > 1 and/or sampled alts are not included) + melt_df[alt_id_name] = melt_df.groupby([chooser_name, variable_column])[ + alt_id_name + ].cumcount() + melt_df = melt_df.set_index( [chooser_name, variable_column, alt_id_name] ).unstack(2) From cd3d07ea4fdad4883716117cb3519b2d38274276 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Wed, 25 Sep 2024 16:17:38 -0700 Subject: [PATCH 09/11] sorting dest choice bundles --- activitysim/abm/models/location_choice.py | 11 +++++++---- activitysim/core/estimation.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index 83e794b2b..98c01288c 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -517,10 +517,13 @@ def run_location_sample( ["person_id", "alt_dest", "prob", "pick_count"] ].set_index("person_id") choices = choices.append(new_choices, ignore_index=False).sort_index() - # making probability the mean of all other sampled destinations by person - # FIXME is there a better way to do this? Does this even matter for estimation? - choices["prob"] = choices["prob"].fillna( - choices.groupby("person_id")["prob"].transform("mean") + # making prob 0 for missing rows so it does not influence model decision + choices["prob"] = choices["prob"].fillna(0) + # sort by person_id and alt_dest + choices = ( + choices.reset_index() + .sort_values(by=["person_id", "alt_dest"]) + .set_index("person_id") ) return choices diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py index 793f1e532..84c9562ea 100644 --- a/activitysim/core/estimation.py +++ b/activitysim/core/estimation.py @@ -571,7 +571,7 @@ def melt_alternatives(self, df): # mergesort is the only stable sort, and we want the expressions to appear in original df column order melt_df = ( pd.melt(df, id_vars=[chooser_name, alt_id_name]) - .sort_values(by=chooser_name, kind="mergesort") + .sort_values(by=[chooser_name, alt_id_name, "variable"], kind="mergesort") .rename(columns={"variable": variable_column}) ) From 8a1fa3c808aff85432ca568b62bc24addb05c6ee Mon Sep 17 00:00:00 2001 From: David Hensle Date: Wed, 25 Sep 2024 16:18:22 -0700 Subject: [PATCH 10/11] adding coalesce edbs as its own step --- activitysim/core/steps/output.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/activitysim/core/steps/output.py b/activitysim/core/steps/output.py index d4ebccdb3..e8b5a4658 100644 --- a/activitysim/core/steps/output.py +++ b/activitysim/core/steps/output.py @@ -259,7 +259,7 @@ def concat_and_write_edb(df_concat_dict, write_dir): raise ValueError(f"Unknown file type {table_name}") -def coalesce_estimation_data_bundles(state): +def _coalesce_estimation_data_bundles(state): """ In estimation mode, estimation data bundles are written to separate subdirectories for each subprocess. This model will go through each subdirectory and move the files to the parent directory. @@ -330,7 +330,7 @@ def coalesce_estimation_data_bundles(state): or is_size_terms_file ) - if is_duplicate_file and not is_same_edb: + if is_duplicate_file and not os.path.exists(os.path.join(cur_edb, file)): # copy the file to the parent directory shutil.copy(file_path, os.path.join(cur_edb, file)) @@ -572,6 +572,21 @@ def map_func(x): else: raise ValueError(f"unknown file_type {file_type}") + +@workflow.step +def coalesce_estimation_data_bundles(state: workflow.State) -> None: + """ + In estimation mode, estimation data bundles are written to separate subdirectories for each subprocess. + This model will go through each subdirectory and concat / copy the files to the parent directory. + This will only occur if the lowest level directory contains the multiprocess step names. + Only multiprocess step names are used because that's how EDBs are written in estimation mode. + + """ is_estimation = estimation_enabled(state) if state.settings.multiprocess and is_estimation: - coalesce_estimation_data_bundles(state) + _coalesce_estimation_data_bundles(state) + else: + logger.info( + "Not in estimation mode or not using multiprocess. Nothing to coalesce." + ) + return From e8c03e6b6fc95a5762ca06f435a2affe35e8b6f1 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 27 Sep 2024 16:45:52 -0700 Subject: [PATCH 11/11] CI testing initial commit --- .github/workflows/core_tests.yml | 53 ++++ activitysim/core/estimation.py | 8 +- activitysim/core/steps/output.py | 5 +- .../configs_estimation/.gitignore | 2 + .../estimation_template.yaml | 129 ++++++++ .../configs_estimation/logging.yaml | 67 +++++ .../configs_estimation/settings_template.yaml | 165 +++++++++++ .../test/test_edb_creation/outputs/.gitignore | 1 + .../survey_data/final_households.csv | 51 ++++ .../final_joint_tour_participants.csv | 8 + .../survey_data/final_persons.csv | 91 ++++++ .../survey_data/final_tours.csv | 118 ++++++++ .../survey_data/final_trips.csv | 278 ++++++++++++++++++ .../survey_data/override_households.csv | 51 ++++ .../override_joint_tour_participants.csv | 8 + .../survey_data/override_persons.csv | 91 ++++++ .../survey_data/override_tours.csv | 118 ++++++++ .../survey_data/override_trips.csv | 278 ++++++++++++++++++ .../survey_data/survey_households.csv | 51 ++++ .../survey_joint_tour_participants.csv | 8 + .../survey_data/survey_persons.csv | 91 ++++++ .../survey_data/survey_tours.csv | 118 ++++++++ .../survey_data/survey_trips.csv | 278 ++++++++++++++++++ .../test_edb_creation/test_edb_formation.py | 263 +++++++++++++++++ .../example_estimation/scripts/infer.py | 12 +- 25 files changed, 2338 insertions(+), 5 deletions(-) create mode 100644 activitysim/estimation/test/test_edb_creation/configs_estimation/.gitignore create mode 100644 activitysim/estimation/test/test_edb_creation/configs_estimation/estimation_template.yaml create mode 100644 activitysim/estimation/test/test_edb_creation/configs_estimation/logging.yaml create mode 100644 activitysim/estimation/test/test_edb_creation/configs_estimation/settings_template.yaml create mode 100644 activitysim/estimation/test/test_edb_creation/outputs/.gitignore create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/final_households.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/final_joint_tour_participants.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/final_persons.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/final_tours.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/final_trips.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/override_households.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/override_joint_tour_participants.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/override_persons.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/override_tours.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/override_trips.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/survey_households.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/survey_joint_tour_participants.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/survey_persons.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/survey_tours.csv create mode 100644 activitysim/estimation/test/test_edb_creation/survey_data/survey_trips.csv create mode 100644 activitysim/estimation/test/test_edb_creation/test_edb_formation.py diff --git a/.github/workflows/core_tests.yml b/.github/workflows/core_tests.yml index 134242cd6..6c95140e4 100644 --- a/.github/workflows/core_tests.yml +++ b/.github/workflows/core_tests.yml @@ -438,6 +438,59 @@ jobs: - name: Test Estimation Mode run: | python -m pytest activitysim/estimation/test/test_larch_estimation.py --durations=0 + + + estimation_edb_creation: + needs: foundation + env: + python-version: "3.10" + label: linux-64 + defaults: + run: + shell: bash -l {0} + name: estimation_edb_creation_test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Mambaforge + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + activate-environment: asim-test + use-mamba: true + python-version: ${{ env.python-version }} + + - name: Set cache date for year and month + run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV + + - uses: actions/cache@v4 + with: + path: ${{ env.CONDA }}/envs + key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }} + id: cache + + - name: Update environment + run: | + mamba env update -n asim-test -f conda-environments/github-actions-tests.yml + if: steps.cache.outputs.cache-hit != 'true' + + - name: Install activitysim + # installing without dependencies is faster, we trust that all needed dependencies + # are in the conda environment defined above. Also, this avoids pip getting + # confused and reinstalling tables (pytables). + run: | + python -m pip install -e . --no-deps + + - name: Conda checkup + run: | + mamba info -a + mamba list + + - name: Test Estimation EDB Creation + run: | + python -m pytest activitysim/estimation/test/test_edb_creation/test_edb_formation.py --durations=0 develop-docbuild: needs: foundation diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py index 84c9562ea..b79618509 100644 --- a/activitysim/core/estimation.py +++ b/activitysim/core/estimation.py @@ -73,12 +73,11 @@ def __init__( os.makedirs(output_dir) # make directory if needed # delete estimation files - unlink_files(self.output_directory(), file_types=("csv", "yaml", "parquet")) + unlink_files(self.output_directory()) if self.bundle_name != self.model_name: # kind of inelegant to always delete these, but ok as they are redundantly recreated for each sub model unlink_files( self.output_directory(bundle_directory=True), - file_types=("csv", "yaml", "parquet"), ) # FIXME - not required? @@ -243,6 +242,9 @@ def write_pickle(self, df, file_path, index, append=False): # therefore we are resetting the index into a column if we want to keep it # if we don't want to keep it, we are dropping it on write with index=False if index: + if df.index.name in df.columns: + # replace old index with new one + df.drop(columns=[df.index.name], inplace=True) df = df.reset_index(drop=False) else: df = df.reset_index(drop=True) @@ -251,7 +253,7 @@ def write_pickle(self, df, file_path, index, append=False): append == True ), f"file already exists: {file_path}" - if append: + if append and os.path.isfile(file_path): # read the previous df and concat prev_df = pd.read_pickle(file_path) df = pd.concat([prev_df, df]) diff --git a/activitysim/core/steps/output.py b/activitysim/core/steps/output.py index e8b5a4658..887503347 100644 --- a/activitysim/core/steps/output.py +++ b/activitysim/core/steps/output.py @@ -304,7 +304,7 @@ def _coalesce_estimation_data_bundles(state): if ( (not is_same_edb) and (len(df_concat_dict) > 0) - and (len(df_concat_dict[list(df_concat_dict.keys())[0]]) > 1) + # and (len(df_concat_dict[list(df_concat_dict.keys())[0]]) > 1) ): concat_and_write_edb(df_concat_dict, prev_edb) @@ -313,6 +313,9 @@ def _coalesce_estimation_data_bundles(state): df_concat_dict = {} for i, file in enumerate(os.listdir(dir)): + + if "stop_frequency" in file: + print("debugging") # get the file path file_path = os.path.join(dir, file) diff --git a/activitysim/estimation/test/test_edb_creation/configs_estimation/.gitignore b/activitysim/estimation/test/test_edb_creation/configs_estimation/.gitignore new file mode 100644 index 000000000..e767d25ce --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/configs_estimation/.gitignore @@ -0,0 +1,2 @@ +settings.yaml +estimation.yaml \ No newline at end of file diff --git a/activitysim/estimation/test/test_edb_creation/configs_estimation/estimation_template.yaml b/activitysim/estimation/test/test_edb_creation/configs_estimation/estimation_template.yaml new file mode 100644 index 000000000..7bf21b93b --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/configs_estimation/estimation_template.yaml @@ -0,0 +1,129 @@ +EDB_ALTS_FILE_FORMAT: compact +EDB_FILETYPE: csv + +enable: True + +bundles: + - school_location + - workplace_location + - auto_ownership + - vehicle_type_choice + - free_parking + - cdap + - mandatory_tour_frequency + - mandatory_tour_scheduling_work + - mandatory_tour_scheduling_school + - joint_tour_frequency + - joint_tour_composition + - joint_tour_participation + - joint_tour_destination + - joint_tour_scheduling + - non_mandatory_tour_frequency + - non_mandatory_tour_destination + - non_mandatory_tour_scheduling + - tour_mode_choice + - atwork_subtour_frequency + - atwork_subtour_destination + - atwork_subtour_scheduling + - atwork_subtour_mode_choice + - stop_frequency + - trip_purpose + - trip_destination + - trip_scheduling + - trip_mode_choice + +# - atwork_subtour_mode_choice subtours.tour_mode + +survey_tables: + households: + file_name: override_households.csv + index_col: household_id + persons: + file_name: override_persons.csv + index_col: person_id + tours: + file_name: override_tours.csv + index_col: tour_id + joint_tour_participants: + file_name: override_joint_tour_participants.csv + index_col: participant_id + trips: + file_name: override_trips.csv + index_col: trip_id + +estimation_table_recipes: + + interaction_sample_simulate: + omnibus_tables: + choosers_combined: + - choices + - override_choices + - choosers + alternatives_combined: + - interaction_sample_alternatives + - interaction_expression_values + omnibus_tables_append_columns: [choosers_combined] + + interaction_simulate: + omnibus_tables: + choosers_combined: + - choices + - override_choices + - choosers + omnibus_tables_append_columns: [choosers_combined] + + simple_simulate: + omnibus_tables: + values_combined: + - choices + - override_choices + - expression_values + - choosers + omnibus_tables_append_columns: [values_combined] + + cdap_simulate: + omnibus_tables: + values_combined: + - choices + - override_choices + - choosers + omnibus_tables_append_columns: [values_combined] + + simple_probabilistic: + omnibus_tables: + values_combined: + - choices + - override_choices + - choosers + - probs + omnibus_tables_append_columns: [values_combined] + + +model_estimation_table_types: + school_location: interaction_sample_simulate + workplace_location: interaction_sample_simulate + auto_ownership: simple_simulate + vehicle_type_choice: interaction_simulate + free_parking: simple_simulate + cdap: cdap_simulate + mandatory_tour_frequency: simple_simulate + mandatory_tour_scheduling_work: interaction_sample_simulate + mandatory_tour_scheduling_school: interaction_sample_simulate + joint_tour_frequency: simple_simulate + joint_tour_composition: simple_simulate + joint_tour_participation: simple_simulate + joint_tour_destination: interaction_sample_simulate + joint_tour_scheduling: interaction_sample_simulate + non_mandatory_tour_frequency: interaction_simulate + non_mandatory_tour_destination: interaction_sample_simulate + non_mandatory_tour_scheduling: interaction_sample_simulate + tour_mode_choice: simple_simulate + atwork_subtour_frequency: simple_simulate + atwork_subtour_destination: interaction_sample_simulate + atwork_subtour_scheduling: interaction_sample_simulate + atwork_subtour_mode_choice: simple_simulate + stop_frequency: simple_simulate + trip_purpose: simple_probabilistic + trip_destination: interaction_sample_simulate + trip_scheduling: simple_probabilistic + trip_mode_choice: simple_simulate diff --git a/activitysim/estimation/test/test_edb_creation/configs_estimation/logging.yaml b/activitysim/estimation/test/test_edb_creation/configs_estimation/logging.yaml new file mode 100644 index 000000000..f4902943d --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/configs_estimation/logging.yaml @@ -0,0 +1,67 @@ +# Config for logging +# ------------------ +# See http://docs.python.org/2.7/library/logging.config.html#configuration-dictionary-schema + +logging: + version: 1 + disable_existing_loggers: true + + + # Configuring the default (root) logger is highly recommended + root: + level: NOTSET + handlers: [console, logfile, elogfile] + + loggers: + + estimation: + level: DEBUG + handlers: [console, elogfile] + propagate: false + + activitysim: + level: INFO + handlers: [console, logfile] + propagate: false + + orca: + level: WARN + handlers: [console, logfile] + propagate: false + + handlers: + + elogfile: + class: logging.FileHandler + filename: + get_log_file_path: 'estimation.log' + mode: w + formatter: fileFormatter + level: NOTSET + + logfile: + class: logging.FileHandler + filename: + get_log_file_path: 'activitysim.log' + mode: w + formatter: fileFormatter + level: NOTSET + + console: + class: logging.StreamHandler + stream: ext://sys.stdout + formatter: simpleFormatter + level: NOTSET + + formatters: + + simpleFormatter: + class: logging.Formatter + # format: '%(levelname)s - %(name)s - %(message)s' + format: '%(levelname)s - %(message)s' + datefmt: '%d/%m/%Y %H:%M:%S' + + fileFormatter: + class: logging.Formatter + format: '%(asctime)s - %(levelname)s - %(name)s - %(message)s' + datefmt: '%d/%m/%Y %H:%M:%S' diff --git a/activitysim/estimation/test/test_edb_creation/configs_estimation/settings_template.yaml b/activitysim/estimation/test/test_edb_creation/configs_estimation/settings_template.yaml new file mode 100644 index 000000000..75b9720c7 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/configs_estimation/settings_template.yaml @@ -0,0 +1,165 @@ + +inherit_settings: True + +# assume enough RAM to not chunk +chunk_training_mode: disabled + +# input tables +input_table_list: + # + # households (table index 'household_id') + # + - tablename: households + filename: override_households.csv + index_col: household_id + keep_columns: + - home_zone_id + - income + - hhsize + - HHT + - auto_ownership + - num_workers + # + # persons (table index 'person_id') + # + - tablename: persons + filename: override_persons.csv + keep_columns: + - household_id + - age + - PNUM + - sex + - pemploy + - pstudent + - ptype + # + # land_use (table index 'zone_id') + # + - tablename: land_use + filename: land_use.csv + rename_columns: + # accept either TAZ or ZONE (but not both) + TAZ: zone_id + ZONE: zone_id + COUNTY: county_id + keep_columns: + - DISTRICT + - SD + - county_id + - TOTHH + - TOTPOP + - TOTACRE + - RESACRE + - CIACRE + - TOTEMP + - AGE0519 + - RETEMPN + - FPSEMPN + - HEREMPN + - OTHEMPN + - AGREMPN + - MWTEMPN + - PRKCST + - OPRKCST + - area_type + - HSENROLL + - COLLFTE + - COLLPTE + - TOPOLOGY + - TERMINAL + +write_raw_tables: False +rng_base_seed: 0 + +fail_fast: True + +use_shadow_pricing: False + +# turn writing of sample_tables on and off for all models +# (if True, tables will be written if DEST_CHOICE_SAMPLE_TABLE_NAME is specified in individual model settings) +want_dest_choice_sample_tables: False + +# number of households to simulate +households_sample_size: 0 + +# to resume after last successful checkpoint, specify resume_after: _ +#resume_after: initialize_households + +trace_hh_id: + +multiprocess: false +num_processes: 2 + + +output_tables: + h5_store: False + action: include + prefix: final_ + sort: True + tables: + - checkpoints + - accessibility + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + +resume_after: + +models: + - initialize_landuse + - initialize_households + - compute_accessibility + - school_location + - workplace_location + - auto_ownership_simulate + - free_parking + - cdap_simulate + - mandatory_tour_frequency + - mandatory_tour_scheduling + - joint_tour_frequency + - joint_tour_composition + - joint_tour_participation + - joint_tour_destination + - joint_tour_scheduling + - non_mandatory_tour_frequency + - non_mandatory_tour_destination + - non_mandatory_tour_scheduling + - tour_mode_choice_simulate + - atwork_subtour_frequency + - atwork_subtour_destination + - atwork_subtour_scheduling + - atwork_subtour_mode_choice + - stop_frequency + - trip_purpose + - trip_destination +# - trip_purpose_and_destination + - trip_scheduling + - trip_mode_choice +# - write_data_dictionary +# - track_skim_usage +# - write_trip_matrices + - write_tables + - coalesce_estimation_data_bundles + + +multiprocess_steps: + - name: mp_initialize + begin: initialize_landuse + - name: mp_accessibility + begin: compute_accessibility + slice: + tables: + - accessibility + # don't slice any tables not explicitly listed above in slice.tables + exclude: True + - name: mp_households + begin: school_location + slice: + tables: + - households + - persons + - name: mp_summarize + begin: write_tables \ No newline at end of file diff --git a/activitysim/estimation/test/test_edb_creation/outputs/.gitignore b/activitysim/estimation/test/test_edb_creation/outputs/.gitignore new file mode 100644 index 000000000..5ed517c37 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/outputs/.gitignore @@ -0,0 +1 @@ +/output*/ \ No newline at end of file diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/final_households.csv b/activitysim/estimation/test/test_edb_creation/survey_data/final_households.csv new file mode 100644 index 000000000..c91fceec9 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/final_households.csv @@ -0,0 +1,51 @@ +"home_zone_id","income","hhsize","HHT","auto_ownership","num_workers","sample_rate","income_in_thousands","income_segment","median_value_of_time","hh_value_of_time","num_non_workers","num_drivers","num_adults","num_children","num_young_children","num_children_5_to_15","num_children_16_to_17","num_college_age","num_young_adults","non_family","family","home_is_urban","home_is_rural","hh_work_auto_savings_ratio","num_under16_not_at_school","num_travel_active","num_travel_active_adults","num_travel_active_preschoolers","num_travel_active_children","num_travel_active_non_preschoolers","participates_in_jtf_model","joint_tour_frequency","num_hh_joint_tours","household_id" +16,30900,2,5,1,2,0.01,30.9,2,8.81,6.021274002645185,0,2,2,0,0,0,0,0,2,true,false,true,false,0.39972082,0,2,2,0,0,2,true,"0_tours",0,982875 +16,99700,9,2,1,4,0.01,99.7,3,10.44,3.705326363656352,5,8,8,1,1,0,0,3,3,false,true,true,false,0.71195495,0,7,6,1,1,6,true,"0_tours",0,1810015 +20,58160,3,1,1,1,0.01,58.16,2,8.81,10.708809147889093,2,2,2,1,1,0,0,0,0,false,true,true,false,0.2646,0,3,2,1,1,2,true,"0_tours",0,1099626 +6,59220,1,4,1,0,0.01,59.22,2,8.81,10.449019648473794,1,1,1,0,0,0,0,0,0,true,false,true,false,0,0,1,1,0,0,1,false,"0_tours",0,763879 +18,51000,1,4,0,1,0.01,51,2,8.81,17.13534609189038,0,1,1,0,0,0,0,0,1,true,false,true,false,0.18706083,0,1,1,0,0,1,false,"0_tours",0,824207 +8,0,1,0,0,1,0.01,0,1,6.01,7.558195608572484,0,1,1,0,0,0,0,1,0,false,false,true,false,0.20204751,0,1,1,0,0,1,false,"0_tours",0,2822230 +8,0,1,0,0,1,0.01,0,1,6.01,1,0,1,1,0,0,0,0,0,0,false,false,true,false,0.19095585,0,1,1,0,0,1,false,"0_tours",0,2821179 +25,31360,5,1,0,1,0.01,31.36,2,8.81,12.074894163110672,4,2,2,3,2,1,0,0,1,false,true,true,false,0.14576416,0,4,1,2,3,2,true,"0_tours",0,1196298 +24,58300,2,2,1,1,0.01,58.3,2,8.81,10.4823297442415,1,2,2,0,0,0,0,0,0,false,true,true,false,0.12962,0,2,2,0,0,2,true,"0_tours",0,1363467 +16,21000,3,1,1,2,0.01,21,1,6.01,5.098171219655474,1,2,2,1,0,1,0,0,0,false,true,true,false,0.28337085,0,3,2,0,1,3,true,"0_tours",0,386761 +9,133000,2,1,0,2,0.01,133,4,12.86,6.387619611064064,0,2,2,0,0,0,0,0,0,false,true,true,false,0.30083168,0,2,2,0,0,2,true,"0_tours",0,2223027 +7,0,1,0,0,0,0.01,0,1,6.01,3.0402011627479264,1,1,1,0,0,0,0,0,0,false,false,true,false,0,0,1,1,0,0,1,false,"0_tours",0,2832182 +20,118800,2,1,1,2,0.01,118.8,4,12.86,21.606392235487583,0,2,2,0,0,0,0,0,0,false,true,true,false,0.26,0,2,2,0,0,2,true,"0_tours",0,2727273 +14,60000,1,4,0,1,0.01,60,2,8.81,4.0074881716523025,0,1,1,0,0,0,0,0,0,true,false,true,false,0.13325916,0,0,0,0,0,0,false,"0_tours",0,1444715 +16,18000,1,6,1,1,0.01,18,1,6.01,2.647107057372401,0,1,1,0,0,0,0,0,0,true,false,true,false,0.23449998,0,1,1,0,0,1,false,"0_tours",0,112064 +14,61900,1,4,1,1,0.01,61.9,3,10.44,2.5980714149446418,0,1,1,0,0,0,0,0,0,true,false,true,false,0.08287584,0,1,1,0,0,1,false,"0_tours",0,1952792 +16,144100,2,1,0,2,0.01,144.1,4,12.86,7.408554295817758,0,2,2,0,0,0,0,0,2,false,true,true,false,0.4657992,0,2,2,0,0,2,true,"1_Eat",1,2223759 +8,0,1,0,0,1,0.01,0,1,6.01,2.467783203776223,0,1,1,0,0,0,0,1,0,false,false,true,false,0.11770582,0,0,0,0,0,0,false,"0_tours",0,2820538 +10,24000,1,4,0,0,0.01,24,1,6.01,5.584662029883316,1,1,1,0,0,0,0,0,0,true,false,true,false,0,0,1,1,0,0,1,false,"0_tours",0,27726 +21,1500,1,4,0,0,0.01,1.5,1,6.01,10.364201291815256,1,1,1,0,0,0,0,0,0,true,false,true,false,0,0,0,0,0,0,0,false,"0_tours",0,570454 +21,18000,3,2,0,1,0.01,18,1,6.01,4.005080218193281,2,2,2,1,0,1,0,1,0,false,true,true,false,0.1529925,0,1,0,0,1,1,false,"0_tours",0,370497 +8,45000,4,1,1,0,0.01,45,2,8.81,2.9767301704144296,4,2,2,2,0,2,0,0,0,false,true,true,false,0,0,4,2,0,2,4,true,"1_Shop",1,1173905 +11,30000,1,4,1,0,0.01,30,1,6.01,3.6040501851984663,1,1,1,0,0,0,0,0,0,true,false,true,false,0,0,1,1,0,0,1,false,"0_tours",0,1286557 +20,0,1,0,0,0,0.01,0,1,6.01,6.660210105218662,1,1,1,0,0,0,0,0,0,false,false,true,false,0,0,1,1,0,0,1,false,"0_tours",0,2762078 +16,0,1,0,1,0,0.01,0,1,6.01,7.951880936376377,1,1,1,0,0,0,0,0,0,false,false,true,false,0,0,0,0,0,0,0,false,"0_tours",0,2832429 +7,21000,3,1,0,2,0.01,21,1,6.01,4.590650730864872,1,2,2,1,0,1,0,0,0,false,true,true,false,0.2743725,0,3,2,0,1,3,true,"0_tours",0,386699 +8,0,1,0,1,1,0.01,0,1,6.01,10.786630771770398,0,1,1,0,0,0,0,0,0,false,false,true,false,0.10683333,0,1,1,0,0,1,false,"0_tours",0,2822097 +9,34630,1,4,1,0,0.01,34.63,2,8.81,7.593970066915795,1,1,1,0,0,0,0,0,0,true,false,true,false,0,0,1,1,0,0,1,false,"0_tours",0,764150 +8,0,1,0,1,1,0.01,0,1,6.01,1.8304507502262177,0,1,1,0,0,0,0,0,0,false,false,true,false,0.15995501,0,1,1,0,0,1,false,"0_tours",0,2820774 +10,69200,2,1,0,2,0.01,69.2,3,10.44,17.6624037989451,0,2,2,0,0,0,0,0,2,false,true,true,false,0.23833334,0,2,2,0,0,2,true,"0_tours",0,1594621 +16,12000,2,7,0,2,0.01,12,1,6.01,8.417809979092251,0,2,2,0,0,0,0,2,0,true,false,true,false,0.3178517,0,2,2,0,0,2,true,"0_tours",0,257531 +9,92700,2,3,0,0,0.01,92.7,3,10.44,11.518178930487295,2,1,1,1,0,1,0,0,0,false,true,true,false,0,1,2,1,0,1,2,true,"0_tours",0,1645132 +17,4200,1,4,0,0,0.01,4.2,1,6.01,4.768890669889258,1,1,1,0,0,0,0,0,0,true,false,true,false,0,0,1,1,0,0,1,false,"0_tours",0,29625 +25,37200,4,1,1,4,0.01,37.2,2,8.81,10.32100270756728,0,4,4,0,0,0,0,2,0,false,true,true,false,0.56286997,0,4,4,0,0,4,true,"1_Disc",1,1402945 +12,30000,1,4,0,1,0.01,30,1,6.01,4.360890352038947,0,1,1,0,0,0,0,0,0,true,false,true,false,0.06425,0,1,1,0,0,1,false,"0_tours",0,823501 +16,76000,3,1,0,2,0.01,76,3,10.44,15.5812014089547,1,2,2,1,0,1,0,0,0,false,true,true,false,0.3402317,0,3,2,0,1,3,true,"0_tours",0,1747467 +17,68000,1,6,0,1,0.01,68,3,10.44,7.595766438703255,0,1,1,0,0,0,0,0,0,true,false,true,false,0.31015667,0,1,1,0,0,1,false,"0_tours",0,1445222 +8,3500,1,4,0,0,0.01,3.5,1,6.01,41.95868930639066,1,1,1,0,0,0,0,0,0,true,false,true,false,0,0,1,1,0,0,1,false,"0_tours",0,26844 +8,0,1,0,0,1,0.01,0,1,6.01,2.9220038814547133,0,1,1,0,0,0,0,1,0,false,false,true,false,0.12387166,0,1,1,0,0,1,false,"0_tours",0,2822219 +16,5050,1,4,0,1,0.01,5.05,1,6.01,3.9205668642399836,0,1,1,0,0,0,0,0,1,true,false,true,false,0.35884497,0,0,0,0,0,0,false,"0_tours",0,110675 +5,388000,1,4,0,1,0.01,388,4,12.86,6.218529751813423,0,1,1,0,0,0,0,0,0,true,false,true,false,0.26293832,0,1,1,0,0,1,false,"0_tours",0,2048204 +8,34400,1,6,0,0,0.01,34.4,2,8.81,11.48239030277669,1,1,1,0,0,0,0,0,0,true,false,true,false,0,0,1,1,0,0,1,false,"0_tours",0,1286259 +17,8000,1,4,1,0,0.01,8,1,6.01,7.32202829064968,1,1,1,0,0,0,0,0,0,true,false,true,false,0,0,1,1,0,0,1,false,"0_tours",0,568785 +8,0,1,4,0,0,0.01,0,1,6.01,2.84310444200838,1,1,1,0,0,0,0,0,0,true,false,true,false,0,0,1,1,0,0,1,false,"0_tours",0,26686 +8,88600,2,1,0,1,0.01,88.6,3,10.44,4.2960940332239455,1,2,2,0,0,0,0,0,0,false,true,true,false,0.1417767,0,2,2,0,0,2,true,"0_tours",0,1511234 +7,195000,1,4,0,1,0.01,195,4,12.86,8.219354788406312,0,1,1,0,0,0,0,0,1,true,false,true,false,0.15494083,0,1,1,0,0,1,false,"0_tours",0,2048382 +10,27800,2,7,0,2,0.01,27.8,1,6.01,1.5416062020885206,0,2,2,0,0,0,0,2,0,true,false,true,false,0.29182667,0,1,1,0,0,1,false,"0_tours",0,256660 +25,17210,2,1,1,0,0.01,17.21,1,6.01,1.8372202530691708,2,2,2,0,0,0,0,0,0,false,true,true,false,0,0,2,2,0,0,2,true,"0_tours",0,703381 +9,4000,2,1,0,1,0.01,4,1,6.01,2.5758755480394475,1,2,2,0,0,0,0,0,2,false,true,true,false,0.23177332,0,2,2,0,0,2,true,"0_tours",0,226869 +11,50000,1,4,1,1,0.01,50,2,8.81,12.952762239760721,0,1,1,0,0,0,0,0,1,true,false,true,false,0.17517333,0,0,0,0,0,0,false,"0_tours",0,823426 diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/final_joint_tour_participants.csv b/activitysim/estimation/test/test_edb_creation/survey_data/final_joint_tour_participants.csv new file mode 100644 index 000000000..7e7029f73 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/final_joint_tour_participants.csv @@ -0,0 +1,8 @@ +"tour_id","household_id","person_id","participant_num","participant_id" +220958279,2223759,5389226,1,22095827901 +220958279,2223759,5389227,2,22095827902 +100798519,1173905,2458502,1,10079851903 +100798519,1173905,2458503,2,10079851904 +130727777,1402945,3188483,1,13072777702 +130727777,1402945,3188484,2,13072777703 +130727777,1402945,3188485,3,13072777704 diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/final_persons.csv b/activitysim/estimation/test/test_edb_creation/survey_data/final_persons.csv new file mode 100644 index 000000000..edfe67587 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/final_persons.csv @@ -0,0 +1,91 @@ +"household_id","age","PNUM","sex","pemploy","pstudent","ptype","age_16_to_19","age_16_p","adult","male","female","has_non_worker","has_retiree","has_preschool_kid","has_driving_kid","has_school_kid","has_full_time","has_part_time","has_university","student_is_employed","nonstudent_to_school","is_student","is_gradeschool","is_highschool","is_university","school_segment","is_worker","home_zone_id","value_of_time","school_zone_id","school_location_logsum","distance_to_school","roundtrip_auto_time_to_school","workplace_zone_id","workplace_location_logsum","distance_to_work","workplace_in_cbd","work_zone_area_type","roundtrip_auto_time_to_work","work_auto_savings","work_auto_savings_ratio","free_parking_at_work","cdap_activity","travel_active","under16_not_at_school","has_preschool_kid_at_home","has_school_kid_at_home","mandatory_tour_frequency","work_and_school_and_worker","work_and_school_and_student","num_mand","num_work_tours","num_joint_tours","non_mandatory_tour_frequency","num_non_mand","num_escort_tours","num_eatout_tours","num_shop_tours","num_maint_tours","num_discr_tours","num_social_tours","num_non_escort_tours","person_id" +26686,39,1,1,3,3,4,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,8,2.84310444200838,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,12,2,0,1,0,1,0,0,2,26686 +26844,51,1,1,3,3,4,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,8,41.95868930639066,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,2,1,0,0,0,0,0,1,1,26844 +27726,52,1,1,3,3,4,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,10,5.584662029883316,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,1,1,0,0,0,0,1,0,1,27726 +29625,61,1,1,3,3,4,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,17,4.768890669889258,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,16,1,0,0,1,0,0,0,1,29625 +110675,30,1,1,2,3,2,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,16,3.9205668642399836,-1,,,0,19,13.725626729995362,1.6,true,1,9.73,43.061398,0.35884497,false,"H",false,false,false,false,"",false,false,0,0,0,0,0,0,0,0,0,0,0,0,110675 +112064,48,1,2,2,3,2,false,true,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,16,2.647107057372401,-1,,,0,21,15.492083174049057,0.96,true,1,5.26,28.139997,0.23449998,false,"N",true,false,false,false,"",false,false,0,0,0,17,2,0,0,1,0,1,0,2,112064 +226869,28,1,1,2,3,2,false,true,true,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,9,2.5758755480394475,-1,,,0,24,15.354599835527907,1.67,true,0,11.389999,27.812798,0.23177332,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,264107 +226869,27,2,2,3,3,4,false,true,true,false,true,false,false,false,false,false,false,true,false,false,false,false,false,false,false,0,false,9,2.5758755480394475,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,1,1,0,0,0,0,1,0,1,264108 +256660,22,1,2,2,3,2,false,true,true,false,true,false,false,false,false,false,false,true,false,false,false,false,false,false,false,0,true,10,1.5416062020885206,-1,,,0,1,14.098626830690556,1.57,true,0,10.940001,18.8746,0.15728833,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,323689 +256660,23,2,2,2,3,2,false,true,true,false,true,false,false,false,false,false,false,true,false,false,false,false,false,false,false,0,true,10,1.5416062020885206,-1,,,0,5,14.131499655555483,1.14,true,0,8.049999,16.1446,0.13453834,false,"H",false,false,false,false,"",false,false,0,0,0,0,0,0,0,0,0,0,0,0,323690 +257531,22,1,2,2,3,2,false,true,true,false,true,false,false,false,false,false,false,true,false,false,false,false,false,false,false,0,true,16,8.417809979092251,-1,,,0,12,13.802811148472596,0.67,true,0,4.65,17.645102,0.14704251,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,325431 +257531,22,2,2,2,3,2,false,true,true,false,true,false,false,false,false,false,false,true,false,false,false,false,false,false,false,0,true,16,8.417809979092251,-1,,,0,2,13.787334502957885,0.66,true,0,5.0699997,20.4971,0.17080918,false,"N",true,false,false,false,"",false,false,0,0,0,16,1,0,0,1,0,0,0,1,325432 +370497,52,1,1,2,3,2,false,true,true,true,false,false,false,false,true,true,false,false,false,false,false,false,false,false,false,0,true,21,4.005080218193281,-1,,,0,4,15.371088271013553,0.88,true,0,5.3900003,18.3591,0.1529925,false,"H",false,false,false,false,"",false,false,0,0,0,0,0,0,0,0,0,0,0,0,595684 +370497,18,2,1,3,1,6,true,true,true,true,false,false,false,false,false,true,false,true,false,false,false,true,false,true,false,2,false,21,4.005080218193281,13,16.12477815830904,0.76,5.08,-1,,,false,,0,0,0,false,"H",false,false,false,false,"",false,false,0,0,0,0,0,0,0,0,0,0,0,0,595685 +370497,14,3,2,4,1,7,false,false,false,false,true,false,false,false,true,false,false,true,false,false,false,true,true,false,false,1,false,21,2.6713885055349182,8,20.920595339763924,0.71,5.07,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,0,0,0,0,0,0,0,0,0,0,595686 +386699,47,1,1,2,3,2,false,true,true,true,false,false,false,false,false,true,true,false,false,false,false,false,false,false,false,0,true,7,4.590650730864872,-1,,,0,12,15.426483631583249,0.77,true,0,4.81,14.3318,0.11943167,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,644290 +386699,43,2,2,1,3,1,false,true,true,false,true,false,false,false,false,true,false,true,false,false,false,false,false,false,false,0,true,7,4.590650730864872,-1,,,0,2,15.430078506697235,0.96,true,0,5.88,18.5929,0.15494083,false,"M",true,false,false,false,"work1",false,false,1,1,0,8,1,0,0,0,1,0,0,1,644291 +386699,7,3,1,4,1,7,false,false,false,true,false,false,false,false,false,false,true,true,false,false,false,true,true,false,false,1,false,7,3.0619640374868697,9,20.36118907767507,0.86,4.91,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,0,0,0,0,0,0,0,0,0,0,644292 +386761,47,1,1,2,3,2,false,true,true,true,false,false,false,false,false,true,true,false,false,false,false,false,false,false,false,0,true,16,5.098171219655474,-1,,,0,4,15.590597722513333,0.68,true,0,4.63,17.6251,0.14687583,false,"M",true,false,false,false,"work1",false,false,1,1,0,41,3,1,0,0,1,1,0,2,644476 +386761,43,2,2,1,3,1,false,true,true,false,true,false,false,false,false,true,false,true,false,false,false,false,false,false,false,0,true,16,5.098171219655474,-1,,,0,15,15.601574170160411,0.47,true,0,3.3899999,16.379402,0.13649502,false,"N",true,false,false,false,"",false,false,0,0,0,16,1,0,0,1,0,0,0,1,644477 +386761,7,3,1,4,1,7,false,false,false,true,false,false,false,false,false,false,true,true,false,false,false,true,true,false,false,1,false,16,3.4004802035102015,20,19.95067445224191,1.49,8.48,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,0,0,0,0,0,0,0,0,0,0,644478 +568785,80,1,1,3,3,5,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,17,7.32202829064968,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,1,1,0,0,0,0,1,0,1,1265898 +570454,85,1,1,3,3,5,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,21,10.364201291815256,-1,,,0,-1,,,false,,0,0,0,false,"H",false,false,false,false,"",false,false,0,0,0,0,0,0,0,0,0,0,0,0,1267567 +703381,65,1,1,3,3,5,false,true,true,true,false,false,true,false,false,false,false,false,false,false,false,false,false,false,false,0,false,25,1.8372202530691708,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,16,1,0,0,1,0,0,0,1,1427193 +703381,69,2,2,3,3,5,false,true,true,false,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,0,false,25,1.8372202530691708,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,10,2,0,0,0,1,0,1,2,1427194 +763879,60,1,1,3,3,4,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,6,10.449019648473794,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,16,1,0,0,1,0,0,0,1,1572659 +764150,52,1,1,3,3,4,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,9,7.593970066915795,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,24,2,0,0,1,1,0,0,2,1572930 +823426,31,1,1,1,3,1,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,11,12.952762239760721,-1,,,0,2,15.565710510565173,1.02,true,0,6.81,21.0208,0.17517333,false,"H",false,false,false,false,"",false,false,0,0,0,0,0,0,0,0,0,0,0,0,1632206 +823501,36,1,1,1,3,1,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,12,4.360890352038947,-1,,,0,13,15.700363517892352,0.24,true,0,1.89,7.7099996,0.06425,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,1632281 +824207,29,1,1,2,3,2,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,18,17.13534609189038,-1,,,0,4,13.53236354932943,1.26,true,0,7.3900003,22.4473,0.18706083,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,1632987 +982875,25,1,1,1,3,1,false,true,true,true,false,false,false,false,false,false,false,true,true,false,false,false,false,false,false,0,true,16,6.021274002645185,-1,,,0,10,15.590713009893207,1.61,true,0,9.91,30.3414,0.252845,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,1875721 +982875,25,2,2,2,2,3,false,true,true,false,true,false,false,false,false,false,true,false,false,true,false,true,false,false,true,3,true,16,6.021274002645185,13,12.760001421247718,0.46,2.99,4,15.603313124205917,0.68,true,0,4.63,17.6251,0.14687583,false,"N",true,false,false,false,"",false,false,0,0,0,4,1,0,1,0,0,0,0,1,1875722 +1099626,35,1,1,1,3,1,false,true,true,true,false,false,false,true,false,false,false,false,true,false,false,false,false,false,false,0,true,20,10.708809147889093,-1,,,0,2,13.499558028231366,1.63,true,0,10.13,31.752003,0.2646,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,2159057 +1099626,36,2,2,3,2,3,false,true,true,false,true,false,false,true,false,false,true,false,false,false,false,true,false,false,true,3,false,20,10.708809147889093,9,7.716620155624279,0.78,4.6,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,0,0,0,0,0,0,0,0,0,0,2159058 +1099626,3,3,1,4,1,8,false,false,false,true,false,false,false,false,false,false,true,false,true,false,false,true,true,false,false,1,false,20,7.142775701642026,20,10.16596170222939,0.23,1.4,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,0,0,0,0,0,0,0,0,0,0,2159059 +1173905,40,1,1,3,3,4,false,true,true,true,false,true,false,false,false,true,false,false,false,false,false,false,false,false,false,0,false,8,2.9767301704144296,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,8,1,0,0,0,1,0,0,1,2458500 +1173905,42,2,2,3,3,4,false,true,true,false,true,true,false,false,false,true,false,false,false,false,false,false,false,false,false,0,false,8,2.9767301704144296,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,32,1,1,0,0,0,0,0,0,2458501 +1173905,12,3,2,4,1,7,false,false,false,false,true,true,false,false,false,true,false,false,false,false,false,true,true,false,false,1,false,8,1.9854790236664246,8,20.93607187982974,0.12,0.78,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,1,0,0,0,0,1,0,0,0,0,2458502 +1173905,9,4,1,4,1,7,false,false,false,true,false,true,false,false,false,true,false,false,false,false,false,true,true,false,false,1,false,8,1.9854790236664246,8,20.57860073687866,0.12,0.78,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,1,0,0,0,0,0,0,0,0,0,2458503 +1196298,36,1,1,1,3,1,false,true,true,true,false,true,false,true,false,true,false,false,false,false,false,false,false,false,false,0,true,25,12.074894163110672,-1,,,0,1,15.551818554287767,0.73,true,0,5.21,17.4917,0.14576416,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,2566698 +1196298,29,2,2,3,3,4,false,true,true,false,true,false,false,true,false,true,true,false,false,false,false,false,false,false,false,0,false,25,12.074894163110672,-1,,,0,-1,,,false,,0,0,0,false,"H",false,false,false,false,"",false,false,0,0,0,0,0,0,0,0,0,0,0,0,2566699 +1196298,8,3,1,4,1,7,false,false,false,true,false,true,false,true,false,false,true,false,false,false,false,true,true,false,false,1,false,25,8.05395440679482,25,20.139937486434746,0.13,0.86,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,0,0,0,0,0,0,0,0,0,0,2566700 +1196298,4,4,2,4,1,8,false,false,false,false,true,true,false,true,false,true,true,false,false,false,false,true,true,false,false,1,false,25,8.05395440679482,3,20.285545324456304,0.41,3.5700002,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,0,0,0,0,0,0,0,0,0,0,2566701 +1196298,2,5,1,4,1,8,false,false,false,true,false,true,false,true,false,true,true,false,false,false,false,true,true,false,false,1,false,25,8.05395440679482,6,20.162884633511972,0.78,5.63,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,0,0,0,0,0,0,0,0,0,0,2566702 +1286259,67,1,2,3,3,5,false,true,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,8,11.48239030277669,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,17,2,0,0,1,0,1,0,2,2936550 +1286557,67,1,1,3,3,5,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,11,3.6040501851984663,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,5,2,0,1,0,0,1,0,2,2936848 +1363467,68,1,1,3,3,5,false,true,true,true,false,false,false,false,false,false,false,true,false,false,false,false,false,false,false,0,false,24,10.4823297442415,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,16,1,0,0,1,0,0,0,1,3061894 +1363467,63,2,2,2,3,2,false,true,true,false,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,0,true,24,10.4823297442415,-1,,,0,25,13.759441855802429,0.5,true,0,3.52,15.5543995,0.12962,false,"M",true,false,false,false,"work1",false,false,1,1,0,9,2,0,0,0,1,1,0,2,3061895 +1402945,66,1,1,2,3,2,false,true,true,true,false,false,false,false,false,false,false,true,true,false,false,false,false,false,false,0,true,25,10.32100270756728,-1,,,0,24,15.538000311308064,0.48,true,0,3.52,15.5543995,0.12962,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,3188482 +1402945,48,2,2,2,3,2,false,true,true,false,true,false,false,false,false,false,false,true,true,false,false,false,false,false,false,0,true,25,10.32100270756728,-1,,,0,2,15.531149525318941,0.45,true,0,3.4699998,17.33,0.14441666,false,"M",true,false,false,false,"work1",false,false,1,1,1,0,0,0,0,0,0,1,0,0,3188483 +1402945,22,3,2,2,2,3,false,true,true,false,true,false,false,false,false,false,false,true,true,true,false,true,false,false,true,3,true,25,10.32100270756728,13,11.401936182943931,0.91,7.08,2,15.547680085581693,0.45,true,0,3.4699998,17.33,0.14441666,false,"M",true,false,false,false,"work_and_school",true,true,2,1,1,0,0,0,0,0,0,0,0,0,3188484 +1402945,19,4,2,2,2,3,true,true,true,false,true,false,false,false,false,false,false,true,true,true,false,true,false,false,true,3,true,25,10.32100270756728,9,11.406264826287744,1.56,10.530001,2,15.533387802712083,0.45,true,0,3.4699998,17.33,0.14441666,false,"M",true,false,false,false,"work1",false,false,1,1,1,0,0,0,0,0,0,0,0,0,3188485 +1444715,42,1,1,2,3,2,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,14,4.0074881716523025,-1,,,0,8,13.89688636450196,1.18,true,0,8.18,15.9911,0.13325916,false,"H",false,false,false,false,"",false,false,0,0,0,0,0,0,0,0,0,0,0,0,3232955 +1445222,43,1,2,1,3,1,false,true,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,17,7.595766438703255,-1,,,0,22,13.744452615299863,1.36,true,0,7.59,37.2188,0.31015667,false,"M",true,false,false,false,"work1",false,false,1,1,0,17,3,0,0,2,0,1,0,3,3233462 +1511234,53,1,2,1,3,1,false,true,true,false,true,false,false,false,false,false,false,false,true,false,false,false,false,false,false,0,true,8,4.2960940332239455,-1,,,0,1,13.81269859955603,1.05,true,0,7.6400003,17.013203,0.1417767,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,3328568 +1511234,61,2,1,3,2,3,false,true,true,true,false,false,false,false,false,false,true,false,false,false,false,true,false,false,true,3,false,8,4.2960940332239455,13,8.537371920112303,0.91,6.96,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,20,2,0,1,1,0,0,0,2,3328569 +1594621,31,1,1,1,3,1,false,true,true,true,false,false,false,false,false,false,false,true,false,false,false,false,false,false,false,0,true,10,17.6624037989451,-1,,,0,11,15.65673756421631,0.49,true,0,4.61,14.99,0.124916665,false,"M",true,false,false,false,"work1",false,false,1,1,0,1,1,0,0,0,0,1,0,1,3495342 +1594621,25,2,2,2,3,2,false,true,true,false,true,false,false,false,false,false,true,false,false,false,false,false,false,false,false,0,true,10,17.6624037989451,-1,,,0,9,15.663788754130458,0.49,true,0,2.79,13.610002,0.11341668,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,3495343 +1645132,56,1,2,3,2,3,false,true,true,false,true,false,false,false,false,true,false,false,false,false,false,true,false,false,true,3,false,9,11.518178930487295,9,9.065704601782443,0.17,1.08,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,20,2,0,1,1,0,0,0,2,3596364 +1645132,13,2,2,4,1,7,false,false,false,false,true,false,false,false,false,false,false,false,true,false,false,true,true,false,false,1,false,9,7.682625346635026,10,10.997808724651822,0.33,2.79,-1,,,false,,0,0,0,false,"N",true,true,false,false,"",false,false,0,0,0,2,1,0,0,0,0,0,1,1,3596365 +1747467,36,1,2,1,3,1,false,true,true,false,true,false,false,false,false,true,false,true,false,false,false,false,false,false,false,0,true,16,15.5812014089547,-1,,,0,11,15.790768885585281,1.13,true,0,6.85,23.182701,0.19318917,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,3891102 +1747467,67,2,1,2,3,2,false,true,true,true,false,false,false,false,false,true,true,false,false,false,false,false,false,false,false,0,true,16,15.5812014089547,-1,,,0,12,15.810492366417014,0.67,true,0,4.65,17.645102,0.14704251,false,"N",true,false,false,false,"",false,false,0,0,0,16,1,0,0,1,0,0,0,1,3891103 +1747467,8,3,1,4,1,7,false,false,false,true,false,false,false,false,false,false,true,true,false,false,false,true,true,false,false,1,false,16,10.392661339772785,17,20.51958126009443,0.55,3.84,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,0,1,2,0,0,0,0,2,0,2,3891104 +1810015,29,1,1,3,2,3,false,true,true,true,false,true,false,true,true,false,true,true,false,false,false,true,false,false,true,3,false,16,3.705326363656352,12,10.535691561088802,0.67,4.65,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,0,0,0,0,0,0,0,0,0,0,4171615 +1810015,20,2,1,3,3,4,false,true,true,true,false,true,false,true,true,false,true,true,true,false,false,false,false,false,false,0,false,16,3.705326363656352,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,8,1,0,0,0,1,0,0,1,4171616 +1810015,27,3,1,1,3,1,false,true,true,true,false,true,false,true,true,false,true,true,true,false,false,false,false,false,false,0,true,16,3.705326363656352,-1,,,0,15,14.465318636285936,0.47,true,0,3.3899999,16.379402,0.13649502,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,4171617 +1810015,24,4,1,2,3,2,false,true,true,true,false,true,false,true,true,false,true,true,true,false,false,false,false,false,false,0,true,16,3.705326363656352,-1,,,0,13,14.461210466251647,0.46,true,0,2.99,15.41,0.12841667,false,"H",false,false,false,false,"",false,false,0,0,0,0,0,0,0,0,0,0,0,0,4171618 +1810015,58,5,2,3,3,4,false,true,true,false,true,true,false,true,true,false,true,true,true,false,false,false,false,false,false,0,false,16,3.705326363656352,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,16,1,0,0,1,0,0,0,1,4171619 +1810015,3,6,2,4,1,8,false,false,false,false,true,true,false,false,true,false,true,true,true,false,false,true,true,false,false,1,false,16,2.4714526845587867,8,11.178622471450844,1.39,9.309999,-1,,,false,,0,0,0,false,"M",true,false,false,false,"school1",false,false,1,0,0,0,0,0,0,0,0,0,0,0,4171620 +1810015,19,7,2,2,1,6,true,true,true,false,true,true,false,true,false,false,true,true,true,true,false,true,false,true,false,2,true,16,3.705326363656352,13,7.554959896288849,0.46,2.99,22,14.475775585095361,1.01,true,0,5.84,25.505198,0.21254331,false,"H",false,false,false,false,"",false,false,0,0,0,0,0,0,0,0,0,0,0,0,4171621 +1810015,31,8,2,3,3,4,false,true,true,false,true,true,false,true,true,false,true,true,true,false,false,false,false,false,false,0,false,16,3.705326363656352,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,16,1,0,0,1,0,0,0,1,4171622 +1810015,36,9,2,1,3,1,false,true,true,false,true,true,false,true,true,false,true,true,true,false,false,false,false,false,false,0,true,16,3.705326363656352,-1,,,0,21,14.455337491545183,0.96,true,1,5.26,28.139997,0.23449998,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,4171623 +1952792,74,1,1,2,3,2,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,14,2.5980714149446418,-1,,,0,12,15.708211617788441,0.45,true,0,3.15,9.945101,0.08287584,false,"M",true,false,false,false,"work1",false,false,1,1,0,9,2,0,0,0,1,1,0,2,4823797 +2048204,40,1,1,1,3,1,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,5,6.218529751813423,-1,,,0,23,14.004424311191945,1.46,true,1,9.16,31.552597,0.26293832,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,5057160 +2048382,33,1,1,1,3,1,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,7,8.219354788406312,-1,,,0,2,13.895050253457644,0.96,true,0,5.88,18.5929,0.15494083,false,"M",true,false,false,false,"work1",false,false,1,1,0,30,4,0,1,1,1,0,1,4,5057338 +2223027,49,1,1,1,3,1,false,true,true,true,false,false,false,false,false,false,true,false,false,false,false,false,false,false,false,0,true,9,6.387619611064064,-1,,,0,4,13.912084249194642,1.23,true,0,8.16,15.9552,0.13296,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,5387762 +2223027,39,2,2,1,3,1,false,true,true,false,true,false,false,false,false,false,true,false,false,false,false,false,false,false,false,0,true,9,6.387619611064064,-1,,,0,14,13.91744334490833,1.55,true,0,10.71,20.1446,0.16787167,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,5387763 +2223759,28,1,2,1,3,1,false,true,true,false,true,false,false,false,false,false,true,false,false,false,false,false,false,false,false,0,true,16,7.408554295817758,-1,,,0,1,14.01123384048398,0.51,true,0,3.73,17.6825,0.14735417,false,"M",true,false,false,false,"work1",false,false,1,1,1,0,0,0,1,0,0,0,0,0,5389226 +2223759,29,2,1,1,3,1,false,true,true,true,false,false,false,false,false,false,true,false,false,false,false,false,false,false,false,0,true,16,7.408554295817758,-1,,,0,23,14.01521677947659,1.21,true,1,8.01,38.2134,0.31844503,false,"M",true,false,false,false,"work1",false,false,1,1,1,0,0,0,0,0,0,0,0,0,5389227 +2727273,72,1,1,2,3,2,false,true,true,true,false,false,false,false,false,false,false,true,false,false,false,false,false,false,false,0,true,20,21.606392235487583,-1,,,0,20,13.845922250496361,0.23,true,1,1.4,7.7999997,0.065,false,"M",true,false,false,false,"work1",false,false,1,1,0,1,1,0,0,0,0,1,0,1,7305540 +2727273,62,2,2,2,3,2,false,true,true,false,true,false,false,false,false,false,false,true,false,false,false,false,false,false,false,0,true,20,21.606392235487583,-1,,,0,9,13.82908782269542,0.78,true,0,4.6,23.4,0.195,false,"M",true,false,false,false,"work2",false,false,2,2,0,0,0,0,0,0,0,0,0,0,7305541 +2762078,57,1,1,3,3,4,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,20,6.660210105218662,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,16,1,0,0,1,0,0,0,1,7453413 +2820538,18,1,1,2,3,2,true,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,8,2.467783203776223,-1,,,0,11,15.473764868535847,0.54,true,0,3.6,14.124699,0.11770582,false,"H",false,false,false,false,"",false,false,0,0,0,0,0,0,0,0,0,0,0,0,7511873 +2820774,35,1,1,1,3,1,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,8,1.8304507502262177,-1,,,0,2,15.359656301391546,0.99,true,0,7.16,19.194601,0.15995501,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,7512109 +2821179,60,1,1,2,3,2,false,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,8,1,-1,,,0,22,15.316797175665155,1.43,true,0,9.67,22.914701,0.19095585,false,"N",true,false,false,false,"",false,false,0,0,0,22,3,0,1,1,0,0,1,3,7512514 +2822097,39,1,2,2,3,2,false,true,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,true,8,10.786630771770398,-1,,,0,7,15.586470857848573,0.25,true,0,2.38,12.82,0.10683333,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,7513432 +2822219,18,1,2,1,3,1,true,true,true,false,true,false,false,false,false,false,false,false,false,false,true,false,false,false,false,0,true,8,2.9220038814547133,-1,,,0,9,15.428749460641615,0.53,true,0,3.33,14.864599,0.12387166,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,7513554 +2822230,19,1,2,2,2,3,true,true,true,false,true,false,false,false,false,false,false,false,false,true,false,true,false,false,true,3,true,8,7.558195608572484,12,12.656259331864534,0.72,6.04,25,15.543074447856814,1.14,true,0,8.04,24.2457,0.20204751,false,"M",true,false,false,false,"work1",false,false,1,1,0,0,0,0,0,0,0,0,0,0,7513565 +2832182,87,1,2,3,3,5,false,true,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,7,3.0402011627479264,-1,,,0,-1,,,false,,0,0,0,false,"N",true,false,false,false,"",false,false,0,0,0,1,1,0,0,0,0,1,0,1,7523517 +2832429,93,1,2,3,3,5,false,true,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,0,false,16,7.951880936376377,-1,,,0,-1,,,false,,0,0,0,false,"H",false,false,false,false,"",false,false,0,0,0,0,0,0,0,0,0,0,0,0,7523764 diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/final_tours.csv b/activitysim/estimation/test/test_edb_creation/survey_data/final_tours.csv new file mode 100644 index 000000000..8dc5c1206 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/final_tours.csv @@ -0,0 +1,118 @@ +"person_id","tour_type","tour_type_count","tour_type_num","tour_num","tour_count","tour_category","number_of_participants","destination","origin","household_id","tdd","start","end","duration","composition","destination_logsum","tour_mode","mode_choice_logsum","atwork_subtour_frequency","parent_tour_id","stop_frequency","primary_purpose","tour_id" +264107,"work",1,1,1,1,"mandatory",1,24,9,226869,49,7,19,12,"",,"WALK_LRF",5.706465236833864,"no_subtours",,"0out_0in","work",10828426 +323689,"work",1,1,1,1,"mandatory",1,1,10,256660,106,11,18,7,"",,"WALK_LRF",5.762750226520877,"no_subtours",,"0out_0in","work",13271288 +325431,"work",1,1,1,1,"mandatory",1,12,16,257531,47,7,17,10,"",,"TNC_SINGLE",5.877301554634767,"no_subtours",,"1out_3in","work",13342710 +595686,"school",1,1,1,1,"mandatory",1,8,21,370497,46,7,16,9,"",,"WALK_LOC",19.487029458680386,"",,"0out_0in","school",24423157 +644290,"work",1,1,1,1,"mandatory",1,12,7,386699,122,12,22,10,"",,"WALK_LOC",5.610320631120498,"no_subtours",,"0out_0in","work",26415929 +644291,"work",1,1,1,1,"mandatory",1,2,7,386699,47,7,17,10,"",,"WALK",5.650927738864148,"no_subtours",,"0out_0in","work",26415970 +644292,"school",1,1,1,1,"mandatory",1,9,7,386699,130,13,19,6,"",,"WALK_LRF",20.42262619148089,"",,"0out_1in","school",26416003 +644476,"work",1,1,1,1,"mandatory",1,4,16,386761,47,7,17,10,"",,"SHARED3FREE",1.761501686723627,"no_subtours",,"0out_0in","work",26423555 +644478,"school",1,1,1,1,"mandatory",1,20,16,386761,107,11,19,8,"",,"WALK_LRF",2.675051981340837,"",,"0out_0in","school",26423629 +1632281,"work",1,1,1,1,"mandatory",1,13,12,823501,49,7,19,12,"",,"TNC_SINGLE",5.737987317212601,"eat",,"0out_0in","work",66923560 +1632987,"work",1,1,1,1,"mandatory",1,4,18,824207,151,15,21,6,"",,"WALK_LRF",6.1374959346320095,"no_subtours",,"0out_0in","work",66952506 +1875721,"work",1,1,1,1,"mandatory",1,10,16,982875,64,8,18,10,"",,"WALK_LOC",0.9683784523915128,"no_subtours",,"0out_0in","work",76904600 +2159057,"work",1,1,1,1,"mandatory",1,2,20,1099626,48,7,18,11,"",,"WALK_LRF",-0.03845876909164505,"no_subtours",,"0out_1in","work",88521376 +2159058,"school",1,1,1,1,"mandatory",1,9,20,1099626,148,15,18,3,"",,"TAXI",0.525351958951068,"",,"0out_0in","univ",88521409 +2159059,"school",1,1,1,1,"mandatory",1,20,20,1099626,59,8,13,5,"",,"WALK",-0.2634059707184708,"",,"0out_0in","school",88521450 +2458502,"school",1,1,1,1,"mandatory",1,8,8,1173905,64,8,18,10,"",,"WALK",0.1992035348928025,"",,"1out_0in","school",100798613 +2458503,"school",1,1,1,1,"mandatory",1,8,8,1173905,60,8,14,6,"",,"WALK",0.2156875433576043,"",,"0out_0in","school",100798654 +2566698,"work",1,1,1,1,"mandatory",1,1,25,1196298,30,6,17,11,"",,"TNC_SINGLE",5.932545072370928,"no_subtours",,"0out_0in","work",105234657 +2566700,"school",1,1,1,1,"mandatory",1,25,25,1196298,45,7,15,8,"",,"WALK",18.287824829249224,"",,"0out_0in","school",105234731 +2566701,"school",1,1,1,1,"mandatory",1,3,25,1196298,59,8,13,5,"",,"WALK",19.073489619505818,"",,"0out_0in","school",105234772 +2566702,"school",1,1,1,1,"mandatory",1,6,25,1196298,120,12,20,8,"",,"WALK_LOC",18.510662532745897,"",,"0out_1in","school",105234813 +3061895,"work",1,1,1,1,"mandatory",1,25,24,1363467,26,6,13,7,"",,"WALK",0.2804129197266325,"no_subtours",,"0out_0in","work",125537734 +3188482,"work",1,1,1,1,"mandatory",1,24,25,1402945,63,8,17,9,"",,"WALK",2.249899545931642,"no_subtours",,"1out_0in","work",130727801 +3188483,"work",1,1,1,1,"mandatory",1,2,25,1402945,78,9,17,8,"",,"WALK_LOC",2.2185417162398577,"no_subtours",,"0out_0in","work",130727842 +3188484,"work",1,1,1,2,"mandatory",1,2,25,1402945,73,9,12,3,"",,"WALK_LOC",2.240731078612465,"no_subtours",,"0out_0in","work",130727883 +3188484,"school",1,1,2,2,"mandatory",1,13,25,1402945,155,16,17,1,"",,"WALK",3.181510470161417,"",,"0out_2in","univ",130727875 +3188485,"work",1,1,1,1,"mandatory",1,2,25,1402945,96,10,21,11,"",,"BIKE",2.1487334423330005,"no_subtours",,"1out_1in","work",130727924 +3233462,"work",1,1,1,1,"mandatory",1,22,17,1445222,53,7,23,16,"",,"TNC_SINGLE",5.922341860291894,"no_subtours",,"0out_0in","work",132571981 +3328568,"work",1,1,1,1,"mandatory",1,1,8,1511234,30,6,17,11,"",,"WALK_LRF",6.052905188790869,"no_subtours",,"0out_0in","work",136471327 +3495342,"work",1,1,1,1,"mandatory",1,11,10,1594621,46,7,16,9,"",,"WALK_LOC",6.106308966698332,"no_subtours",,"0out_0in","work",143309061 +3495343,"work",1,1,1,1,"mandatory",1,9,10,1594621,128,13,17,4,"",,"WALK",6.301543519645703,"eat",,"0out_0in","work",143309102 +3891102,"work",1,1,1,1,"mandatory",1,11,16,1747467,46,7,16,9,"",,"WALK_LOC",5.87041941077222,"no_subtours",,"0out_0in","work",159535221 +3891104,"school",1,1,1,1,"mandatory",1,17,16,1747467,44,7,14,7,"",,"WALK_LRF",20.477184369166057,"",,"0out_0in","school",159535295 +4171615,"school",1,1,1,1,"mandatory",1,12,16,1810015,124,13,13,0,"",,"WALK",2.9751655865930506,"",,"0out_0in","univ",171036246 +4171617,"work",1,1,1,1,"mandatory",1,15,16,1810015,79,9,18,9,"",,"WALK",1.6497477694001255,"no_subtours",,"1out_0in","work",171036336 +4171620,"school",1,1,1,1,"mandatory",1,8,16,1810015,45,7,15,8,"",,"WALK_LOC",1.0091399254943096,"",,"0out_0in","school",171036451 +4171623,"work",1,1,1,1,"mandatory",1,21,16,1810015,69,8,23,15,"",,"WALK",1.3627881263595452,"eat",,"1out_1in","work",171036582 +4823797,"work",1,1,1,1,"mandatory",1,12,14,1952792,103,11,15,4,"",,"BIKE",-0.4317170835900537,"no_subtours",,"0out_0in","work",197775716 +5057160,"work",1,1,1,1,"mandatory",1,23,5,2048204,43,7,13,6,"",,"TNC_SINGLE",5.760634465125435,"no_subtours",,"0out_0in","work",207343599 +5057338,"work",1,1,1,1,"mandatory",1,2,7,2048382,30,6,17,11,"",,"TNC_SINGLE",5.899143436225317,"no_subtours",,"0out_0in","work",207350897 +5387762,"work",1,1,1,1,"mandatory",1,4,9,2223027,47,7,17,10,"",,"WALK_HVY",6.208077244952496,"no_subtours",,"0out_0in","work",220898281 +5387763,"work",1,1,1,1,"mandatory",1,14,9,2223027,63,8,17,9,"",,"WALK_HVY",6.045289166977812,"no_subtours",,"1out_1in","work",220898322 +5389226,"work",1,1,1,1,"mandatory",1,1,16,2223759,79,9,18,9,"",,"WALK",5.842396021294293,"eat",,"0out_0in","work",220958305 +5389227,"work",1,1,1,1,"mandatory",1,23,16,2223759,33,6,20,14,"",,"WALK",5.866576082823254,"no_subtours",,"0out_0in","work",220958346 +7305540,"work",1,1,1,1,"mandatory",1,20,20,2727273,103,11,15,4,"",,"DRIVEALONEFREE",1.9004730948098723,"no_subtours",,"0out_0in","work",299527179 +7305541,"work",2,1,1,2,"mandatory",1,9,20,2727273,26,6,13,7,"",,"WALK",1.920480893206963,"no_subtours",,"0out_0in","work",299527220 +7305541,"work",2,2,2,2,"mandatory",1,9,20,2727273,139,14,18,4,"",,"WALK",1.9391010189034459,"no_subtours",,"0out_0in","work",299527221 +7512109,"work",1,1,1,1,"mandatory",1,2,8,2820774,65,8,19,11,"",,"WALK",-0.14126603138986757,"eat",,"0out_2in","work",307996508 +7513432,"work",1,1,1,1,"mandatory",1,7,8,2822097,10,5,15,10,"",,"BIKE",0.2843400004991259,"no_subtours",,"0out_0in","work",308050751 +7513554,"work",1,1,1,1,"mandatory",1,9,8,2822219,11,5,16,11,"",,"WALK",5.610133706923624,"no_subtours",,"0out_0in","work",308055753 +7513565,"work",1,1,1,1,"mandatory",1,25,8,2822230,79,9,18,9,"",,"WALK",5.728223421490162,"no_subtours",,"1out_0in","work",308056204 +5389226,"eatout",1,1,1,1,"joint",2,13,16,2223759,180,20,20,0,"adults",15.700851700188215,"WALK",0.10567890761343558,"",,"0out_0in","eatout",220958279 +2458502,"shopping",1,1,1,1,"joint",2,5,8,1173905,54,8,8,0,"children",13.054650480322216,"SHARED2FREE",-0.5111781667673684,"",,"0out_0in","shopping",100798519 +3188483,"othdiscr",1,1,1,1,"joint",3,24,25,1402945,70,9,9,0,"adults",14.684837106261803,"WALK",-2.2633078286768997,"",,"0out_0in","othdiscr",130727777 +26686,"othmaint",1,1,1,2,"non_mandatory",1,9,8,26686,113,12,13,1,"",15.128287261384761,"BIKE",1.8681013354742215,"",,"0out_0in","othmaint",1094154 +26686,"eatout",1,1,2,2,"non_mandatory",1,5,8,26686,175,19,19,0,"",15.839748033438521,"WALK",4.389345460274684,"",,"0out_0in","eatout",1094132 +26844,"social",1,1,1,1,"non_mandatory",1,7,8,26844,58,8,12,4,"",14.80937948982704,"WALK",2.3971996226244454,"",,"0out_1in","social",1100640 +27726,"othdiscr",1,1,1,1,"non_mandatory",1,9,10,27726,56,8,10,2,"",15.385947148348619,"WALK",2.8054735464903384,"",,"0out_0in","othdiscr",1136791 +29625,"shopping",1,1,1,1,"non_mandatory",1,17,17,29625,145,15,15,0,"",14.097979049989863,"WALK",2.197070314899941,"",,"0out_0in","shopping",1214658 +112064,"shopping",1,1,1,2,"non_mandatory",1,3,16,112064,155,16,17,1,"",12.863310046769188,"WALK",-0.45236824043048446,"",,"0out_0in","shopping",4594657 +112064,"othdiscr",1,1,2,2,"non_mandatory",1,21,16,112064,114,12,14,2,"",14.175152333791747,"DRIVEALONEFREE",-0.2059367766627193,"",,"0out_0in","othdiscr",4594649 +264108,"othdiscr",1,1,1,1,"non_mandatory",1,19,9,226869,86,10,11,1,"",15.408950294845221,"WALK",1.5151852085832618,"",,"0out_0in","othdiscr",10828453 +325432,"shopping",1,1,1,1,"non_mandatory",1,14,16,257531,113,12,13,1,"",14.164245179577405,"TAXI",2.245802766560401,"",,"0out_0in","shopping",13342745 +644291,"othmaint",1,1,1,1,"non_mandatory",1,2,7,386699,179,19,23,4,"",15.115958969714466,"TNC_SINGLE",1.8502335907456067,"",,"0out_0in","othmaint",26415959 +644476,"escort",1,1,1,3,"non_mandatory",1,11,16,386761,1,5,6,1,"",12.795273575389468,"TNC_SINGLE",-0.7934297704888018,"",,"0out_0in","escort",26423525 +644476,"othmaint",1,1,2,3,"non_mandatory",1,13,16,386761,170,18,19,1,"",14.239645494444684,"WALK",0.5448752034137055,"",,"0out_0in","othmaint",26423544 +644476,"othdiscr",1,1,3,3,"non_mandatory",1,16,16,386761,169,18,18,0,"",14.73709932540282,"WALK",1.901940250188323,"",,"0out_0in","othdiscr",26423541 +644477,"shopping",1,1,1,1,"non_mandatory",1,16,16,386761,135,14,14,0,"",13.817208967526978,"DRIVEALONEFREE",1.9264575100035397,"",,"0out_0in","shopping",26423590 +1265898,"othdiscr",1,1,1,1,"non_mandatory",1,20,17,568785,81,9,20,11,"",14.038862408841926,"WALK_LRF",-0.5585556103261583,"",,"0out_0in","othdiscr",51901843 +1427193,"shopping",1,1,1,1,"non_mandatory",1,14,25,703381,130,13,19,6,"",12.69331890781131,"WALK",-1.0811369726727758,"",,"0out_0in","shopping",58514946 +1427194,"othmaint",1,1,1,2,"non_mandatory",1,14,25,703381,43,7,13,6,"",13.942750115929014,"WALK",-0.9941329062085063,"",,"0out_0in","othmaint",58514982 +1427194,"social",1,1,2,2,"non_mandatory",1,2,25,703381,160,16,22,6,"",13.77700865100175,"WALK",1.0801844297966243,"",,"1out_3in","social",58514990 +1572659,"shopping",1,1,1,1,"non_mandatory",1,24,6,763879,50,7,20,13,"",12.944687277006528,"WALK",-0.5346597893073812,"",,"2out_2in","shopping",64479052 +1572930,"shopping",1,1,1,2,"non_mandatory",1,7,9,764150,104,11,16,5,"",12.871444186599586,"WALK",-0.15716862120827904,"",,"0out_0in","shopping",64490163 +1572930,"othmaint",1,1,2,2,"non_mandatory",1,9,9,764150,57,8,11,3,"",14.115990694346833,"WALK",0.5775008727356886,"",,"1out_0in","othmaint",64490158 +1875722,"eatout",1,1,1,1,"non_mandatory",1,14,16,982875,92,10,17,7,"",14.382991219857317,"WALK",2.162753491521419,"",,"1out_0in","eatout",76904608 +2458500,"othmaint",1,1,1,1,"non_mandatory",1,7,8,1173905,120,12,20,8,"",14.167855032527061,"WALK",0.38559056447197426,"",,"0out_0in","othmaint",100798528 +2458501,"escort",1,1,1,1,"non_mandatory",1,16,8,1173905,146,15,16,1,"",12.678877131453035,"WALK_LOC",-1.932482226653213,"",,"0out_0in","escort",100798550 +2936550,"shopping",1,1,1,2,"non_mandatory",1,11,8,1286259,113,12,13,1,"",14.329923645050131,"WALK",2.380720675067925,"",,"0out_0in","shopping",120398583 +2936550,"othdiscr",1,1,2,2,"non_mandatory",1,6,8,1286259,138,14,17,3,"",15.563753675610899,"WALK",2.937756877729721,"",,"0out_0in","othdiscr",120398575 +2936848,"othdiscr",1,1,1,2,"non_mandatory",1,11,11,1286557,146,15,16,1,"",14.200096334451901,"DRIVEALONEFREE",0.6742944281956302,"",,"1out_0in","othdiscr",120410793 +2936848,"eatout",1,1,2,2,"non_mandatory",1,9,11,1286557,136,14,15,1,"",13.618742937074572,"DRIVEALONEFREE",0.6916712958119103,"",,"0out_0in","eatout",120410774 +3061894,"shopping",1,1,1,1,"non_mandatory",1,20,24,1363467,113,12,13,1,"",12.936400847846238,"DRIVEALONEFREE",-0.419345072964738,"",,"0out_0in","shopping",125537687 +3061895,"othmaint",1,1,1,2,"non_mandatory",1,7,24,1363467,146,15,16,1,"",14.06003170940918,"WALK",-0.16195911293846704,"",,"0out_0in","othmaint",125537723 +3061895,"othdiscr",1,1,2,2,"non_mandatory",1,9,24,1363467,164,17,19,2,"",14.089291551343601,"WALK_HVY",0.30873285639026604,"",,"0out_0in","othdiscr",125537720 +3233462,"shopping",2,1,1,3,"non_mandatory",1,16,17,1445222,19,6,6,0,"",14.283234710901684,"WALK_LRF",2.5866729381679985,"",,"0out_0in","shopping",132571975 +3233462,"shopping",2,2,2,3,"non_mandatory",1,13,17,1445222,37,7,7,0,"",14.291671919294387,"WALK_LOC",1.7366283594420195,"",,"0out_0in","shopping",132571976 +3233462,"othdiscr",1,1,3,3,"non_mandatory",1,14,17,1445222,37,7,7,0,"",15.3475060062512,"WALK_LRF",2.5704471563279228,"",,"0out_0in","othdiscr",132571967 +3328569,"shopping",1,1,1,2,"non_mandatory",1,4,8,1511234,85,10,10,0,"",14.125733968656233,"WALK_LRF",2.38356074513733,"",,"1out_0in","shopping",136471362 +3328569,"eatout",1,1,2,2,"non_mandatory",1,13,8,1511234,137,14,16,2,"",15.597840009148284,"WALK_LRF",3.672048876588288,"",,"0out_0in","eatout",136471335 +3495342,"othdiscr",1,1,1,1,"non_mandatory",1,15,10,1594621,160,16,22,6,"",15.508505478543945,"TNC_SINGLE",2.3148522693391618,"",,"0out_0in","othdiscr",143309047 +3596364,"shopping",1,1,1,2,"non_mandatory",1,22,9,1645132,176,19,20,1,"",14.363714390192708,"WALK_LRF",2.3452602538930165,"",,"0out_2in","shopping",147450957 +3596364,"eatout",1,1,2,2,"non_mandatory",1,5,9,1645132,135,14,14,0,"",15.40958622642079,"WALK",3.8915091275806684,"",,"0out_0in","eatout",147450930 +3596365,"social",1,1,1,1,"non_mandatory",1,9,9,1645132,48,7,18,11,"",14.542235377439113,"TAXI",1.8197268535726026,"",,"0out_0in","social",147451001 +3891103,"shopping",1,1,1,1,"non_mandatory",1,16,16,1747467,127,13,16,3,"",14.345132902591203,"WALK",2.410899830438007,"",,"0out_0in","shopping",159535256 +3891104,"othdiscr",2,1,1,2,"non_mandatory",1,2,16,1747467,136,14,15,1,"",15.50230775562424,"WALK",2.706012062287055,"",,"0out_0in","othdiscr",159535289 +3891104,"othdiscr",2,2,2,2,"non_mandatory",1,19,16,1747467,159,16,21,5,"",15.496659202926004,"WALK_LOC",1.4568065331937983,"",,"0out_0in","othdiscr",159535290 +4171616,"othmaint",1,1,1,1,"non_mandatory",1,3,16,1810015,72,9,11,2,"",14.289824837336404,"WALK",0.18665328796358552,"",,"0out_0in","othmaint",171036284 +4171619,"shopping",1,1,1,1,"non_mandatory",1,1,16,1810015,126,13,15,2,"",13.640567650511668,"WALK",0.9902753937389079,"",,"0out_1in","shopping",171036412 +4171622,"shopping",1,1,1,1,"non_mandatory",1,19,16,1810015,76,9,15,6,"",13.569454077979632,"WALK",-0.6999014421153607,"",,"0out_0in","shopping",171036535 +4823797,"othmaint",1,1,1,2,"non_mandatory",1,7,14,1952792,145,15,15,0,"",14.077945771016731,"DRIVEALONEFREE",0.09809762893266673,"",,"0out_0in","othmaint",197775705 +4823797,"othdiscr",1,1,2,2,"non_mandatory",1,16,14,1952792,156,16,18,2,"",14.148349302926023,"WALK",0.3783571737852308,"",,"0out_0in","othdiscr",197775702 +5057338,"shopping",1,1,1,4,"non_mandatory",1,25,7,2048382,184,21,21,0,"",14.255205007211757,"WALK_LOC",2.101189166034452,"",,"0out_0in","shopping",207350891 +5057338,"othmaint",1,1,2,4,"non_mandatory",1,7,7,2048382,169,18,18,0,"",15.398681601999414,"WALK",2.251684017727115,"",,"0out_0in","othmaint",207350886 +5057338,"eatout",1,1,3,4,"non_mandatory",1,10,7,2048382,180,20,20,0,"",15.673355298407246,"WALK_LRF",3.815469226012807,"",,"0out_0in","eatout",207350864 +5057338,"social",1,1,4,4,"non_mandatory",1,6,7,2048382,170,18,19,1,"",14.470705576395908,"TNC_SINGLE",2.092887419244588,"",,"0out_0in","social",207350894 +7305540,"othdiscr",1,1,1,1,"non_mandatory",1,12,20,2727273,151,15,21,6,"",14.631393485472051,"WALK",0.703051223685245,"",,"0out_0in","othdiscr",299527165 +7453413,"shopping",1,1,1,1,"non_mandatory",1,19,20,2762078,154,16,16,0,"",13.96782273443678,"WALK",1.8263069979086812,"",,"0out_0in","shopping",305589966 +7512514,"shopping",1,1,1,3,"non_mandatory",1,6,8,2821179,164,17,19,2,"",13.761715627405522,"WALK",1.777915016124194,"",,"0out_0in","shopping",308013107 +7512514,"eatout",1,1,2,3,"non_mandatory",1,13,8,2821179,62,8,16,8,"",15.672503858155086,"WALK",3.4520753953932997,"",,"0out_0in","eatout",308013080 +7512514,"social",1,1,3,3,"non_mandatory",1,9,8,2821179,154,16,16,0,"",13.986153838122636,"WALK_LOC",1.0884935003375935,"",,"0out_0in","social",308013110 +7523517,"othdiscr",1,1,1,1,"non_mandatory",1,9,7,2832182,59,8,13,5,"",15.215939189600421,"WALK_LOC",2.3087491550483623,"",,"0out_0in","othdiscr",308464222 +1632281,"eat",1,1,1,1,"atwork",1,8,13,823501,125,13,14,1,"",15.947502350236292,"WALK",5.269628441312229,"",66923560,"1out_0in","atwork",66923525 +3495343,"eat",1,1,1,1,"atwork",1,16,9,1594621,135,14,14,0,"",15.54410431803647,"WALK",4.189553929647487,"",143309102,"3out_0in","atwork",143309067 +4171623,"eat",1,1,1,1,"atwork",1,7,21,1810015,85,10,10,0,"",12.96349084109601,"WALK",-0.21208714075218113,"",171036582,"0out_1in","atwork",171036547 +5389226,"eat",1,1,1,1,"atwork",1,2,1,2223759,124,13,13,0,"",15.712981796209124,"WALK",6.339068060891636,"",220958305,"0out_0in","atwork",220958270 +7512109,"eat",1,1,1,1,"atwork",1,8,2,2820774,85,10,10,0,"",20.633672564068426,"WALK",-0.4712382999322044,"",307996508,"0out_1in","atwork",307996473 diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/final_trips.csv b/activitysim/estimation/test/test_edb_creation/survey_data/final_trips.csv new file mode 100644 index 000000000..e7075be5e --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/final_trips.csv @@ -0,0 +1,278 @@ +"person_id","household_id","primary_purpose","trip_num","outbound","trip_count","destination","origin","tour_id","purpose","destination_logsum","depart","trip_mode","mode_choice_logsum","trip_id" +26686,26686,"eatout",1,true,1,5,8,1094132,"eatout",,19,"WALK",4.457539268772517,8753057 +26686,26686,"eatout",1,false,1,8,5,1094132,"home",,19,"WALK",4.457539268092902,8753061 +26686,26686,"othmaint",1,true,1,9,8,1094154,"othmaint",,12,"BIKE",6.898149355436168,8753233 +26686,26686,"othmaint",1,false,1,8,9,1094154,"home",,13,"WALK",6.898149355330578,8753237 +26844,26844,"social",1,true,1,7,8,1100640,"social",,8,"WALK",8.97158319245669,8805121 +26844,26844,"social",1,false,2,7,7,1100640,"othmaint",43.13165717387488,12,"WALK",9.041583184270134,8805125 +26844,26844,"social",2,false,2,8,7,1100640,"home",,12,"WALK",8.789583223221504,8805126 +27726,27726,"othdiscr",1,true,1,9,10,1136791,"othdiscr",,8,"WALK",10.602053625992742,9094329 +27726,27726,"othdiscr",1,false,1,10,9,1136791,"home",,10,"WALK",10.780613609497287,9094333 +29625,29625,"shopping",1,true,1,17,17,1214658,"shopping",,15,"WALK",6.232122380537646,9717265 +29625,29625,"shopping",1,false,1,17,17,1214658,"home",,15,"WALK",6.232122380537646,9717269 +112064,112064,"othdiscr",1,true,1,21,16,4594649,"othdiscr",,12,"WALK",2.1507947003355725,36757193 +112064,112064,"othdiscr",1,false,1,16,21,4594649,"home",,14,"TAXI",2.361929055638584,36757197 +112064,112064,"shopping",1,true,1,3,16,4594657,"shopping",,16,"WALK",9.171843929425059,36757257 +112064,112064,"shopping",1,false,1,16,3,4594657,"home",,17,"WALK",8.920744017488841,36757261 +264107,226869,"work",1,true,1,24,9,10828426,"work",,7,"WALK_LRF",2.4072481834219825,86627409 +264107,226869,"work",1,false,1,9,24,10828426,"home",,19,"WALK_LRF",2.64270709783459,86627413 +264108,226869,"othdiscr",1,true,1,19,9,10828453,"othdiscr",,10,"WALK",-0.8882719505294207,86627625 +264108,226869,"othdiscr",1,false,1,9,19,10828453,"home",,11,"WALK",-1.083555087607077,86627629 +323689,256660,"work",1,true,1,1,10,13271288,"work",,11,"WALK_LRF",-0.011824703285233131,106170305 +323689,256660,"work",1,false,1,10,1,13271288,"home",,18,"WALK_LRF",-0.027352899414421275,106170309 +325431,257531,"work",1,true,2,25,16,13342710,"shopping",35.382763983172914,7,"WALK",10.195186294548499,106741681 +325431,257531,"work",2,true,2,12,25,13342710,"work",,8,"WALK",4.112435828979324,106741682 +325431,257531,"work",1,false,4,7,12,13342710,"eatout",38.563608123329665,17,"WALK",4.429198666616662,106741685 +325431,257531,"work",2,false,4,25,7,13342710,"shopping",49.069979336099856,17,"WALK_LOC",10.976833414968677,106741686 +325431,257531,"work",3,false,4,7,25,13342710,"escort",49.56683087980666,17,"WALK_LOC",10.645246813510491,106741687 +325431,257531,"work",4,false,4,16,7,13342710,"home",,17,"WALK",10.872836065437152,106741688 +325432,257531,"shopping",1,true,1,14,16,13342745,"shopping",,12,"WALK_LOC",1.5232643877229,106741961 +325432,257531,"shopping",1,false,1,16,14,13342745,"home",,13,"WALK_LOC",1.5693861133527638,106741965 +595686,370497,"school",1,true,1,8,21,24423157,"school",,7,"WALK_LOC",11.180762378631227,195385257 +595686,370497,"school",1,false,1,21,8,24423157,"home",,16,"WALK",11.113688275374244,195385261 +644290,386699,"work",1,true,1,12,7,26415929,"work",,12,"WALK_LOC",4.308779715277221,211327433 +644290,386699,"work",1,false,1,7,12,26415929,"home",,22,"WALK_LOC",4.332367558204261,211327437 +644291,386699,"othmaint",1,true,1,2,7,26415959,"othmaint",,19,"WALK_LOC",0.1926592887611779,211327673 +644291,386699,"othmaint",1,false,1,7,2,26415959,"home",,23,"TNC_SINGLE",0.2090608444298338,211327677 +644291,386699,"work",1,true,1,2,7,26415970,"work",,7,"WALK",-0.6047304841612396,211327761 +644291,386699,"work",1,false,1,7,2,26415970,"home",,17,"WALK",-0.5343416483077198,211327765 +644292,386699,"school",1,true,1,9,7,26416003,"school",,13,"WALK_LOC",9.76993799435511,211328025 +644292,386699,"school",1,false,2,7,9,26416003,"othdiscr",54.47702761519171,19,"WALK_LRF",11.717824347550579,211328029 +644292,386699,"school",2,false,2,7,7,26416003,"home",,19,"WALK",12.412237340788142,211328030 +644476,386761,"escort",1,true,1,11,16,26423525,"escort",,5,"WALK_LOC",4.78915841238078,211388201 +644476,386761,"escort",1,false,1,16,11,26423525,"home",,6,"WALK_LOC",5.050171287433509,211388205 +644476,386761,"othdiscr",1,true,1,16,16,26423541,"othdiscr",,18,"WALK",7.330879513166791,211388329 +644476,386761,"othdiscr",1,false,1,16,16,26423541,"home",,18,"WALK",7.330879513166791,211388333 +644476,386761,"othmaint",1,true,1,13,16,26423544,"othmaint",,18,"WALK",-0.4192505336997586,211388353 +644476,386761,"othmaint",1,false,1,16,13,26423544,"home",,19,"WALK",-0.41925030619506426,211388357 +644476,386761,"work",1,true,1,4,16,26423555,"work",,7,"SHARED2FREE",0.6984134734828816,211388441 +644476,386761,"work",1,false,1,16,4,26423555,"home",,17,"SHARED3FREE",-0.05867188910234502,211388445 +644477,386761,"shopping",1,true,1,16,16,26423590,"shopping",,14,"WALK",5.857453125014367,211388721 +644477,386761,"shopping",1,false,1,16,16,26423590,"home",,14,"WALK",5.857453125014367,211388725 +644478,386761,"school",1,true,1,20,16,26423629,"school",,11,"WALK_LOC",1.530044582582947,211389033 +644478,386761,"school",1,false,1,16,20,26423629,"home",,19,"WALK_LRF",3.76602489416099,211389037 +1265898,568785,"othdiscr",1,true,1,20,17,51901843,"othdiscr",,9,"WALK_LRF",2.7426144387101616,415214745 +1265898,568785,"othdiscr",1,false,1,17,20,51901843,"home",,20,"WALK_LRF",3.0312322271177115,415214749 +1427193,703381,"shopping",1,true,1,14,25,58514946,"shopping",,13,"WALK",0.41368191525132,468119569 +1427193,703381,"shopping",1,false,1,25,14,58514946,"home",,19,"WALK",0.20722357704976505,468119573 +1427194,703381,"othmaint",1,true,1,14,25,58514982,"othmaint",,7,"WALK",0.25954887271815646,468119857 +1427194,703381,"othmaint",1,false,1,25,14,58514982,"home",,13,"WALK",0.13005163957603907,468119861 +1427194,703381,"social",1,true,2,6,25,58514990,"eatout",23.804031444441073,16,"WALK",7.665909210409199,468119921 +1427194,703381,"social",2,true,2,2,6,58514990,"social",,19,"WALK",-0.11373619351391898,468119922 +1427194,703381,"social",1,false,4,6,2,58514990,"social",25.675224145070427,21,"WALK",-0.11373628500729509,468119925 +1427194,703381,"social",2,false,4,8,6,58514990,"shopping",36.086366516709475,22,"WALK",7.728909184292301,468119926 +1427194,703381,"social",3,false,4,7,8,58514990,"social",40.832427907262876,22,"WALK",7.751367569789609,468119927 +1427194,703381,"social",4,false,4,25,7,58514990,"home",,22,"WALK",8.429083086926186,468119928 +1572659,763879,"shopping",1,true,3,25,6,64479052,"othmaint",40.895784325733594,7,"WALK",12.896301701456215,515832417 +1572659,763879,"shopping",2,true,3,25,25,64479052,"escort",40.13139614585728,12,"WALK",13.621701652814899,515832418 +1572659,763879,"shopping",3,true,3,24,25,64479052,"shopping",,17,"WALK",3.0930067693134005,515832419 +1572659,763879,"shopping",1,false,3,25,24,64479052,"shopping",38.41713719577139,18,"WALK",3.0706867412992,515832421 +1572659,763879,"shopping",2,false,3,7,25,64479052,"escort",59.6309483835455,20,"WALK",12.807021629683366,515832422 +1572659,763879,"shopping",3,false,3,6,7,64479052,"home",,20,"WALK",14.258626224164276,515832423 +1572930,764150,"othmaint",1,true,2,7,9,64490158,"othmaint",38.76005910239585,8,"WALK",8.642583134383138,515921265 +1572930,764150,"othmaint",2,true,2,9,7,64490158,"othmaint",,8,"WALK",6.391034808672385,515921266 +1572930,764150,"othmaint",1,false,1,9,9,64490158,"home",,11,"WALK",6.874034571771074,515921269 +1572930,764150,"shopping",1,true,1,7,9,64490163,"shopping",,11,"WALK",13.778746190542957,515921305 +1572930,764150,"shopping",1,false,1,9,7,64490163,"home",,16,"WALK",13.622506184583505,515921309 +1632281,823501,"atwork",1,true,2,9,13,66923525,"work",52.01208618984001,13,"WALK",9.392493718811465,535388201 +1632281,823501,"atwork",2,true,2,8,9,66923525,"atwork",,13,"WALK",12.238334146563448,535388202 +1632281,823501,"atwork",1,false,1,13,8,66923525,"work",,14,"WALK",11.952574143723762,535388205 +1632281,823501,"work",1,true,1,13,12,66923560,"work",,7,"WALK",0.3452900411345135,535388481 +1632281,823501,"work",1,false,1,12,13,66923560,"home",,19,"WALK",0.29278490089518977,535388485 +1632987,824207,"work",1,true,1,4,18,66952506,"work",,15,"WALK_LOC",0.2992185756062765,535620049 +1632987,824207,"work",1,false,1,18,4,66952506,"home",,21,"WALK",0.5557332757868809,535620053 +1875721,982875,"work",1,true,1,10,16,76904600,"work",,8,"WALK_LOC",7.627291076037503,615236801 +1875721,982875,"work",1,false,1,16,10,76904600,"home",,18,"WALK_LOC",7.619393573928726,615236805 +1875722,982875,"eatout",1,true,2,7,16,76904608,"escort",33.332775271121825,10,"WALK",12.852466196970816,615236865 +1875722,982875,"eatout",2,true,2,14,7,76904608,"eatout",,13,"WALK",0.0679239371174617,615236866 +1875722,982875,"eatout",1,false,1,16,14,76904608,"home",,17,"WALK",0.9383092208675533,615236869 +2159057,1099626,"work",1,true,1,2,20,88521376,"work",,7,"WALK",-0.3540033864901755,708171009 +2159057,1099626,"work",1,false,2,8,2,88521376,"shopping",28.059656557964445,18,"WALK",0.34307389812569966,708171013 +2159057,1099626,"work",2,false,2,20,8,88521376,"home",,18,"WALK_LOC",9.930931452887558,708171014 +2159058,1099626,"univ",1,true,1,9,20,88521409,"univ",,15,"WALK_LOC",10.081589126967758,708171273 +2159058,1099626,"univ",1,false,1,20,9,88521409,"home",,18,"WALK_LOC",9.700222902924416,708171277 +2159059,1099626,"school",1,true,1,20,20,88521450,"school",,8,"WALK",2.001157626801728,708171601 +2159059,1099626,"school",1,false,1,20,20,88521450,"home",,13,"WALK",2.001157626801728,708171605 +2458502,1173905,"shopping",1,true,1,5,8,100798519,"shopping",,8,"SHARED2FREE",0.16957406666783342,806388153 +2458502,1173905,"shopping",1,false,1,8,5,100798519,"home",,8,"SHARED2FREE",0.1684460835975957,806388157 +2458500,1173905,"othmaint",1,true,1,7,8,100798528,"othmaint",,12,"WALK",8.971583091962668,806388225 +2458500,1173905,"othmaint",1,false,1,8,7,100798528,"home",,20,"WALK",8.789583098654154,806388229 +2458501,1173905,"escort",1,true,1,16,8,100798550,"escort",,15,"WALK_LOC",6.451456988932713,806388401 +2458501,1173905,"escort",1,false,1,8,16,100798550,"home",,16,"WALK_LOC",6.446188145342157,806388405 +2458502,1173905,"school",1,true,2,7,8,100798613,"social",55.65769426652772,8,"WALK",13.486637386215971,806388905 +2458502,1173905,"school",2,true,2,8,7,100798613,"school",,8,"WALK",11.315206418159061,806388906 +2458502,1173905,"school",1,false,1,8,8,100798613,"home",,18,"WALK",11.737966424070635,806388909 +2458503,1173905,"school",1,true,1,8,8,100798654,"school",,8,"WALK",11.737966424070635,806389233 +2458503,1173905,"school",1,false,1,8,8,100798654,"home",,14,"WALK",11.737966424070635,806389237 +2566698,1196298,"work",1,true,1,1,25,105234657,"work",,6,"WALK",0.5218384234138416,841877257 +2566698,1196298,"work",1,false,1,25,1,105234657,"home",,17,"WALK_LOC",0.4855336440096437,841877261 +2566700,1196298,"school",1,true,1,25,25,105234731,"school",,7,"WALK",12.824615869979219,841877849 +2566700,1196298,"school",1,false,1,25,25,105234731,"home",,15,"WALK",12.824615869979219,841877853 +2566701,1196298,"school",1,true,1,3,25,105234772,"school",,8,"WALK",8.979312480941104,841878177 +2566701,1196298,"school",1,false,1,25,3,105234772,"home",,13,"WALK",8.979312481086987,841878181 +2566702,1196298,"school",1,true,1,6,25,105234813,"school",,12,"WALK_LOC",11.70939586566571,841878505 +2566702,1196298,"school",1,false,2,25,6,105234813,"shopping",51.23589608925055,20,"WALK_LOC",11.23832543650178,841878509 +2566702,1196298,"school",2,false,2,25,25,105234813,"home",,20,"WALK",11.641815891720018,841878510 +2936550,1286259,"othdiscr",1,true,1,6,8,120398575,"othdiscr",,14,"WALK",12.612248980790358,963188601 +2936550,1286259,"othdiscr",1,false,1,8,6,120398575,"home",,17,"WALK",12.322089000527528,963188605 +2936550,1286259,"shopping",1,true,1,11,8,120398583,"shopping",,12,"WALK",4.840354366429668,963188665 +2936550,1286259,"shopping",1,false,1,8,11,120398583,"home",,13,"WALK",4.940794055372203,963188669 +2936848,1286557,"eatout",1,true,1,9,11,120410774,"eatout",,14,"WALK",9.079286100640175,963286193 +2936848,1286557,"eatout",1,false,1,11,9,120410774,"home",,15,"WALK",9.101568714168469,963286197 +2936848,1286557,"othdiscr",1,true,2,10,11,120410793,"escort",33.67980057651356,15,"WALK",8.178730791217355,963286345 +2936848,1286557,"othdiscr",2,true,2,11,10,120410793,"othdiscr",,16,"WALK",3.366293797308929,963286346 +2936848,1286557,"othdiscr",1,false,1,11,11,120410793,"home",,16,"WALK",3.65295752128872,963286349 +3061894,1363467,"shopping",1,true,1,20,24,125537687,"shopping",,12,"TNC_SHARED",0.10121955466253282,1004301497 +3061894,1363467,"shopping",1,false,1,24,20,125537687,"home",,13,"DRIVEALONEFREE",0.019236377431838474,1004301501 +3061895,1363467,"othdiscr",1,true,1,9,24,125537720,"othdiscr",,17,"WALK_HVY",11.684658026322639,1004301761 +3061895,1363467,"othdiscr",1,false,1,24,9,125537720,"home",,19,"WALK_LRF",11.49938905905555,1004301765 +3061895,1363467,"othmaint",1,true,1,7,24,125537723,"othmaint",,15,"WALK",8.131583343724042,1004301785 +3061895,1363467,"othmaint",1,false,1,24,7,125537723,"home",,16,"WALK",8.096583148991245,1004301789 +3061895,1363467,"work",1,true,1,25,24,125537734,"work",,6,"WALK",10.08552703351978,1004301873 +3061895,1363467,"work",1,false,1,24,25,125537734,"home",,13,"WALK",10.103127058632895,1004301877 +3188483,1402945,"othdiscr",1,true,1,24,25,130727777,"othdiscr",,9,"WALK",3.118711021135967,1045822217 +3188483,1402945,"othdiscr",1,false,1,25,24,130727777,"home",,9,"WALK",3.0963906344964696,1045822221 +3188482,1402945,"work",1,true,2,7,25,130727801,"work",34.564591307437105,8,"WALK",10.411761425965901,1045822409 +3188482,1402945,"work",2,true,2,24,7,130727801,"work",,9,"WALK",1.211444903376802,1045822410 +3188482,1402945,"work",1,false,1,25,24,130727801,"home",,17,"WALK",2.091377723899355,1045822413 +3188483,1402945,"work",1,true,1,2,25,130727842,"work",,9,"WALK",0.7316381092221191,1045822737 +3188483,1402945,"work",1,false,1,25,2,130727842,"home",,17,"WALK_LOC",0.6404115318186951,1045822741 +3188484,1402945,"univ",1,true,1,13,25,130727875,"univ",,16,"WALK",-1.5428778623620805,1045823001 +3188484,1402945,"univ",1,false,3,7,13,130727875,"shopping",30.41829198797714,17,"WALK",-1.4237492795265192,1045823005 +3188484,1402945,"univ",2,false,3,7,7,130727875,"othmaint",59.4490857231952,17,"WALK",13.5950373888898,1045823006 +3188484,1402945,"univ",3,false,3,25,7,130727875,"home",,17,"WALK",12.646537396714635,1045823007 +3188484,1402945,"work",1,true,1,2,25,130727883,"work",,9,"WALK",0.7316381092221191,1045823065 +3188484,1402945,"work",1,false,1,25,2,130727883,"home",,12,"WALK_LOC",0.5898827035401254,1045823069 +3188485,1402945,"work",1,true,2,25,25,130727924,"escort",29.779997197410285,10,"BIKE",10.416625527744298,1045823393 +3188485,1402945,"work",2,true,2,2,25,130727924,"work",,12,"BIKE",-0.019493094347113764,1045823394 +3188485,1402945,"work",1,false,2,7,2,130727924,"eatout",29.723531295459807,20,"BIKE",-0.21951892594579855,1045823397 +3188485,1402945,"work",2,false,2,25,7,130727924,"home",,21,"BIKE",10.648166757292298,1045823398 +3233462,1445222,"othdiscr",1,true,1,14,17,132571967,"othdiscr",,7,"WALK",1.8799351940528457,1060575737 +3233462,1445222,"othdiscr",1,false,1,17,14,132571967,"home",,7,"WALK_LOC",1.6549807020273366,1060575741 +3233462,1445222,"shopping",1,true,1,16,17,132571975,"shopping",,6,"WALK",7.8891489495492335,1060575801 +3233462,1445222,"shopping",1,false,1,17,16,132571975,"home",,6,"WALK_LRF",7.904205187610422,1060575805 +3233462,1445222,"shopping",1,true,1,13,17,132571976,"shopping",,7,"WALK_LOC",-0.15795552230629617,1060575809 +3233462,1445222,"shopping",1,false,1,17,13,132571976,"home",,7,"WALK_LOC",-0.22691611232387354,1060575813 +3233462,1445222,"work",1,true,1,22,17,132571981,"work",,7,"WALK_LRF",1.5282754769935067,1060575849 +3233462,1445222,"work",1,false,1,17,22,132571981,"home",,23,"TNC_SINGLE",1.793267333094202,1060575853 +3328568,1511234,"work",1,true,1,1,8,136471327,"work",,6,"WALK_LRF",0.24234305546243562,1091770617 +3328568,1511234,"work",1,false,1,8,1,136471327,"home",,17,"WALK",0.17155654756009,1091770621 +3328569,1511234,"eatout",1,true,1,13,8,136471335,"eatout",,14,"WALK",0.5773709562181402,1091770681 +3328569,1511234,"eatout",1,false,1,8,13,136471335,"home",,16,"WALK_LRF",1.098907001128995,1091770685 +3328569,1511234,"shopping",1,true,2,9,8,136471362,"eatout",33.74288275937864,10,"WALK_LOC",11.097623867544948,1091770897 +3328569,1511234,"shopping",2,true,2,4,9,136471362,"shopping",,10,"WALK_LRF",1.1309568013629243,1091770898 +3328569,1511234,"shopping",1,false,1,8,4,136471362,"home",,10,"WALK_LRF",1.3002912188200697,1091770901 +3495342,1594621,"othdiscr",1,true,1,15,10,143309047,"othdiscr",,16,"WALK_LOC",1.6922670564497784,1146472377 +3495342,1594621,"othdiscr",1,false,1,10,15,143309047,"home",,22,"WALK_LRF",1.7951585505312424,1146472381 +3495342,1594621,"work",1,true,1,11,10,143309061,"work",,7,"WALK_LOC",4.611533756385394,1146472489 +3495342,1594621,"work",1,false,1,10,11,143309061,"home",,16,"WALK",4.609280148452949,1146472493 +3495343,1594621,"atwork",1,true,4,6,9,143309067,"othmaint",43.892801127843825,14,"WALK",12.430369125011044,1146472537 +3495343,1594621,"atwork",2,true,4,6,6,143309067,"shopping",43.85860241624266,14,"WALK",13.016929006188157,1146472538 +3495343,1594621,"atwork",3,true,4,7,6,143309067,"escort",46.55936296536758,14,"WALK",14.326586254989872,1146472539 +3495343,1594621,"atwork",4,true,4,16,7,143309067,"atwork",,14,"WALK",5.875801157805464,1146472540 +3495343,1594621,"atwork",1,false,1,9,16,143309067,"work",,14,"WALK",4.620305346152653,1146472541 +3495343,1594621,"work",1,true,1,9,10,143309102,"work",,13,"WALK",8.030042709603043,1146472817 +3495343,1594621,"work",1,false,1,10,9,143309102,"home",,17,"WALK",8.170842677581534,1146472821 +3596364,1645132,"eatout",1,true,1,5,9,147450930,"eatout",,14,"WALK",3.8995482482297024,1179607441 +3596364,1645132,"eatout",1,false,1,9,5,147450930,"home",,14,"WALK",3.983247379767863,1179607445 +3596364,1645132,"shopping",1,true,1,22,9,147450957,"shopping",,19,"WALK_LRF",2.6756101195264512,1179607657 +3596364,1645132,"shopping",1,false,3,8,22,147450957,"shopping",37.58009946460086,20,"WALK_LRF",2.758604912440526,1179607661 +3596364,1645132,"shopping",2,false,3,9,8,147450957,"eatout",54.10980461901021,20,"WALK_LOC",12.741255996041916,1179607662 +3596364,1645132,"shopping",3,false,3,9,9,147450957,"home",,20,"WALK",10.764874611676083,1179607663 +3596365,1645132,"social",1,true,1,9,9,147451001,"social",,7,"WALK",3.544669911588107,1179608009 +3596365,1645132,"social",1,false,1,9,9,147451001,"home",,18,"WALK",3.5446075289165297,1179608013 +3891102,1747467,"work",1,true,1,11,16,159535221,"work",,7,"WALK_LOC",4.3180888605508825,1276281769 +3891102,1747467,"work",1,false,1,16,11,159535221,"home",,16,"WALK_LOC",4.308039287554886,1276281773 +3891103,1747467,"shopping",1,true,1,16,16,159535256,"shopping",,13,"WALK",7.330879778846902,1276282049 +3891103,1747467,"shopping",1,false,1,16,16,159535256,"home",,16,"WALK",7.330879779011079,1276282053 +3891104,1747467,"othdiscr",1,true,1,2,16,159535289,"othdiscr",,14,"WALK",-0.013620982982507432,1276282313 +3891104,1747467,"othdiscr",1,false,1,16,2,159535289,"home",,15,"WALK",-0.20327093589265946,1276282317 +3891104,1747467,"othdiscr",1,true,1,19,16,159535290,"othdiscr",,16,"WALK_LOC",-0.028685356425070176,1276282321 +3891104,1747467,"othdiscr",1,false,1,16,19,159535290,"home",,21,"WALK_LOC",-0.2990566938111557,1276282325 +3891104,1747467,"school",1,true,1,17,16,159535295,"school",,7,"WALK_LRF",6.698072453217584,1276282361 +3891104,1747467,"school",1,false,1,16,17,159535295,"home",,14,"WALK_LRF",6.665980554341056,1276282365 +4171615,1810015,"univ",1,true,1,12,16,171036246,"univ",,13,"WALK",4.061179288088942,1368289969 +4171615,1810015,"univ",1,false,1,16,12,171036246,"home",,13,"WALK",4.06117926693834,1368289973 +4171616,1810015,"othmaint",1,true,1,3,16,171036284,"othmaint",,9,"WALK",5.752949863933666,1368290273 +4171616,1810015,"othmaint",1,false,1,16,3,171036284,"home",,11,"WALK",5.595449988691705,1368290277 +4171617,1810015,"work",1,true,2,8,16,171036336,"social",23.477345398553453,9,"WALK",7.896576327414681,1368290689 +4171617,1810015,"work",2,true,2,15,8,171036336,"work",,12,"WALK",-0.8211572450364255,1368290690 +4171617,1810015,"work",1,false,1,16,15,171036336,"home",,18,"WALK",0.23912905823533456,1368290693 +4171619,1810015,"shopping",1,true,1,1,16,171036412,"shopping",,13,"WALK",-1.0053143437541998,1368291297 +4171619,1810015,"shopping",1,false,2,7,1,171036412,"shopping",29.48393750646727,14,"WALK",-1.5297238905680486,1368291301 +4171619,1810015,"shopping",2,false,2,16,7,171036412,"home",,15,"WALK",12.573466151788852,1368291302 +4171620,1810015,"school",1,true,1,8,16,171036451,"school",,7,"WALK_LOC",10.68060996983474,1368291609 +4171620,1810015,"school",1,false,1,16,8,171036451,"home",,15,"WALK_LOC",10.665116381563836,1368291613 +4171622,1810015,"shopping",1,true,1,19,16,171036535,"shopping",,9,"WALK",-2.394141994327624,1368292281 +4171622,1810015,"shopping",1,false,1,16,19,171036535,"home",,15,"WALK",-2.4219133842589526,1368292285 +4171623,1810015,"atwork",1,true,1,7,21,171036547,"atwork",,10,"WALK",13.897946303660285,1368292377 +4171623,1810015,"atwork",1,false,2,6,7,171036547,"othmaint",62.239483838845736,10,"WALK",14.364186248721689,1368292381 +4171623,1810015,"atwork",2,false,2,21,6,171036547,"work",,10,"WALK",12.200629295549986,1368292382 +4171623,1810015,"work",1,true,2,25,16,171036582,"escort",30.234430836012045,8,"WALK",9.029527074456235,1368292657 +4171623,1810015,"work",2,true,2,21,25,171036582,"work",,8,"WALK",2.001441630738205,1368292658 +4171623,1810015,"work",1,false,2,7,21,171036582,"work",34.72578612209499,23,"WALK",2.5646380272501803,1368292661 +4171623,1810015,"work",2,false,2,16,7,171036582,"home",,23,"WALK",9.584561374619746,1368292662 +4823797,1952792,"othdiscr",1,true,1,16,14,197775702,"othdiscr",,16,"WALK",6.996079351207767,1582205617 +4823797,1952792,"othdiscr",1,false,1,14,16,197775702,"home",,18,"WALK",7.1523193586223615,1582205621 +4823797,1952792,"othmaint",1,true,1,7,14,197775705,"othmaint",,15,"WALK",7.130522181030106,1582205641 +4823797,1952792,"othmaint",1,false,1,14,7,197775705,"home",,15,"WALK",6.973127233279991,1582205645 +4823797,1952792,"work",1,true,1,12,14,197775716,"work",,11,"BIKE",3.423281973078554,1582205729 +4823797,1952792,"work",1,false,1,14,12,197775716,"home",,15,"BIKE",3.4232819641055308,1582205733 +5057160,2048204,"work",1,true,1,23,5,207343599,"work",,7,"WALK_LRF",1.8355968767710429,1658748793 +5057160,2048204,"work",1,false,1,5,23,207343599,"home",,13,"WALK_LOC",1.842947720747105,1658748797 +5057338,2048382,"eatout",1,true,1,10,7,207350864,"eatout",,20,"WALK",11.50286632419377,1658806913 +5057338,2048382,"eatout",1,false,1,7,10,207350864,"home",,20,"WALK",11.0200368363894,1658806917 +5057338,2048382,"othmaint",1,true,1,7,7,207350886,"othmaint",,18,"WALK",9.041583134997413,1658807089 +5057338,2048382,"othmaint",1,false,1,7,7,207350886,"home",,18,"WALK",9.041583134997413,1658807093 +5057338,2048382,"shopping",1,true,1,25,7,207350891,"shopping",,21,"WALK",12.994038320235191,1658807129 +5057338,2048382,"shopping",1,false,1,7,25,207350891,"home",,21,"WALK",13.394104919134762,1658807133 +5057338,2048382,"social",1,true,1,6,7,207350894,"social",,18,"WALK_LOC",7.684253228078017,1658807153 +5057338,2048382,"social",1,false,1,7,6,207350894,"home",,19,"WALK_LOC",7.33402873870212,1658807157 +5057338,2048382,"work",1,true,1,2,7,207350897,"work",,6,"WALK",0.6535205296575887,1658807177 +5057338,2048382,"work",1,false,1,7,2,207350897,"home",,17,"WALK_LOC",0.6994347038785153,1658807181 +5387762,2223027,"work",1,true,1,4,9,220898281,"work",,7,"WALK_HVY",0.8851251729940038,1767186249 +5387762,2223027,"work",1,false,1,9,4,220898281,"home",,17,"WALK_LRF",0.8808191020800481,1767186253 +5387763,2223027,"work",1,true,2,7,9,220898322,"escort",33.115120502248885,8,"WALK",11.45812100506256,1767186577 +5387763,2223027,"work",2,true,2,14,7,220898322,"work",,8,"WALK_LOC",1.0900866492604258,1767186578 +5387763,2223027,"work",1,false,2,9,14,220898322,"othmaint",32.045095679314514,17,"WALK_LRF",1.600735778924674,1767186581 +5387763,2223027,"work",2,false,2,9,9,220898322,"home",,17,"WALK",8.75484607474322,1767186582 +5389226,2223759,"atwork",1,true,1,2,1,220958270,"atwork",,13,"WALK",0.5422067947215797,1767666161 +5389226,2223759,"atwork",1,false,1,1,2,220958270,"work",,13,"WALK",0.44445625677441347,1767666165 +5389226,2223759,"eatout",1,true,1,13,16,220958279,"eatout",,20,"TNC_SINGLE",-0.053423172630162746,1767666233 +5389226,2223759,"eatout",1,false,1,16,13,220958279,"home",,20,"TAXI",-0.053623759329434514,1767666237 +5389226,2223759,"work",1,true,1,1,16,220958305,"work",,9,"WALK",-1.121725266718706,1767666441 +5389226,2223759,"work",1,false,1,16,1,220958305,"home",,18,"WALK",-1.2712056171232493,1767666445 +5389227,2223759,"work",1,true,1,23,16,220958346,"work",,6,"WALK",0.7526667284662408,1767666769 +5389227,2223759,"work",1,false,1,16,23,220958346,"home",,20,"WALK",0.40072266203872847,1767666773 +7305540,2727273,"othdiscr",1,true,1,12,20,299527165,"othdiscr",,15,"WALK",3.8685996173550423,2396217321 +7305540,2727273,"othdiscr",1,false,1,20,12,299527165,"home",,21,"WALK",3.5896037288320892,2396217325 +7305540,2727273,"work",1,true,1,20,20,299527179,"work",,11,"WALK",1.104068131386293,2396217433 +7305540,2727273,"work",1,false,1,20,20,299527179,"home",,15,"WALK",1.053200360752169,2396217437 +7305541,2727273,"work",1,true,1,9,20,299527220,"work",,6,"WALK",7.774842814489909,2396217761 +7305541,2727273,"work",1,false,1,20,9,299527220,"home",,13,"WALK",7.915642733452835,2396217765 +7305541,2727273,"work",1,true,1,9,20,299527221,"work",,14,"WALK",7.774842814936839,2396217769 +7305541,2727273,"work",1,false,1,20,9,299527221,"home",,18,"WALK",7.91564273351867,2396217773 +7453413,2762078,"shopping",1,true,1,19,20,305589966,"shopping",,16,"WALK",-0.35226970200906427,2444719729 +7453413,2762078,"shopping",1,false,1,20,19,305589966,"home",,16,"WALK",-0.30764319648513383,2444719733 +7512109,2820774,"atwork",1,true,1,8,2,307996473,"atwork",,10,"WALK",11.717494374110142,2463971785 +7512109,2820774,"atwork",1,false,2,7,8,307996473,"work",60.69572620723573,10,"WALK",12.448894080618153,2463971789 +7512109,2820774,"atwork",2,false,2,2,7,307996473,"work",,10,"WALK",13.86034632280424,2463971790 +7512109,2820774,"work",1,true,1,2,8,307996508,"work",,8,"WALK",-0.6314685269146993,2463972065 +7512109,2820774,"work",1,false,3,8,2,307996508,"shopping",25.44599794712168,18,"WALK",-0.7722559635889226,2463972069 +7512109,2820774,"work",2,false,3,8,8,307996508,"shopping",46.27968294620515,19,"WALK",9.5289762031843,2463972070 +7512109,2820774,"work",3,false,3,8,8,307996508,"home",,19,"WALK",9.5289762031843,2463972071 +7512514,2821179,"eatout",1,true,1,13,8,308013080,"eatout",,8,"WALK",-1.171759971785514,2464104641 +7512514,2821179,"eatout",1,false,1,8,13,308013080,"home",,16,"WALK",-1.238718768693438,2464104645 +7512514,2821179,"shopping",1,true,1,6,8,308013107,"shopping",,17,"WALK",12.612248978887928,2464104857 +7512514,2821179,"shopping",1,false,1,8,6,308013107,"home",,19,"WALK",12.322088998148224,2464104861 +7512514,2821179,"social",1,true,1,9,8,308013110,"social",,16,"WALK",6.292424410910544,2464104881 +7512514,2821179,"social",1,false,1,8,9,308013110,"home",,16,"WALK_LOC",6.322192231184283,2464104885 +7513432,2822097,"work",1,true,1,7,8,308050751,"work",,5,"BIKE",11.002889267549957,2464406009 +7513432,2822097,"work",1,false,1,8,7,308050751,"home",,15,"BIKE",10.880760381185553,2464406013 +7513554,2822219,"work",1,true,1,9,8,308055753,"work",,5,"WALK",7.994842572799317,2464446025 +7513554,2822219,"work",1,false,1,8,9,308055753,"home",,16,"WALK",7.994842572767652,2464446029 +7513565,2822230,"work",1,true,2,9,8,308056204,"univ",40.040196758213916,9,"WALK",7.9948426686587775,2464449633 +7513565,2822230,"work",2,true,2,25,9,308056204,"work",,9,"WALK",8.34752700809022,2464449634 +7513565,2822230,"work",1,false,1,8,25,308056204,"home",,18,"WALK",9.31552710851258,2464449637 +7523517,2832182,"othdiscr",1,true,1,9,7,308464222,"othdiscr",,8,"WALK",10.67469554537436,2467713777 +7523517,2832182,"othdiscr",1,false,1,7,9,308464222,"home",,13,"WALK",10.735094428557618,2467713781 diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/override_households.csv b/activitysim/estimation/test/test_edb_creation/survey_data/override_households.csv new file mode 100644 index 000000000..b0910c786 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/override_households.csv @@ -0,0 +1,51 @@ +household_id,home_zone_id,income,hhsize,HHT,auto_ownership,num_workers,joint_tour_frequency +982875,16,30900,2,5,1,2,0_tours +1810015,16,99700,9,2,1,4,0_tours +1099626,20,58160,3,1,1,1,0_tours +763879,6,59220,1,4,1,0,0_tours +824207,18,51000,1,4,0,1,0_tours +2822230,8,0,1,0,0,1,0_tours +2821179,8,0,1,0,0,1,0_tours +1196298,25,31360,5,1,0,1,0_tours +1363467,24,58300,2,2,1,1,0_tours +386761,16,21000,3,1,1,2,0_tours +2223027,9,133000,2,1,0,2,0_tours +2832182,7,0,1,0,0,0,0_tours +2727273,20,118800,2,1,1,2,0_tours +1444715,14,60000,1,4,0,1,0_tours +112064,16,18000,1,6,1,1,0_tours +1952792,14,61900,1,4,1,1,0_tours +2223759,16,144100,2,1,0,2,1_Eat +2820538,8,0,1,0,0,1,0_tours +27726,10,24000,1,4,0,0,0_tours +570454,21,1500,1,4,0,0,0_tours +370497,21,18000,3,2,0,1,0_tours +1173905,8,45000,4,1,1,0,1_Shop +1286557,11,30000,1,4,1,0,0_tours +2762078,20,0,1,0,0,0,0_tours +2832429,16,0,1,0,1,0,0_tours +386699,7,21000,3,1,0,2,0_tours +2822097,8,0,1,0,1,1,0_tours +764150,9,34630,1,4,1,0,0_tours +2820774,8,0,1,0,1,1,0_tours +1594621,10,69200,2,1,0,2,0_tours +257531,16,12000,2,7,0,2,0_tours +1645132,9,92700,2,3,0,0,0_tours +29625,17,4200,1,4,0,0,0_tours +1402945,25,37200,4,1,1,4,1_Disc +823501,12,30000,1,4,0,1,0_tours +1747467,16,76000,3,1,0,2,0_tours +1445222,17,68000,1,6,0,1,0_tours +26844,8,3500,1,4,0,0,0_tours +2822219,8,0,1,0,0,1,0_tours +110675,16,5050,1,4,0,1,0_tours +2048204,5,388000,1,4,0,1,0_tours +1286259,8,34400,1,6,0,0,0_tours +568785,17,8000,1,4,1,0,0_tours +26686,8,0,1,4,0,0,0_tours +1511234,8,88600,2,1,0,1,0_tours +2048382,7,195000,1,4,0,1,0_tours +256660,10,27800,2,7,0,2,0_tours +703381,25,17210,2,1,1,0,0_tours +226869,9,4000,2,1,0,1,0_tours +823426,11,50000,1,4,1,1,0_tours diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/override_joint_tour_participants.csv b/activitysim/estimation/test/test_edb_creation/survey_data/override_joint_tour_participants.csv new file mode 100644 index 000000000..a19d8616a --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/override_joint_tour_participants.csv @@ -0,0 +1,8 @@ +survey_participant_id,survey_tour_id,household_id,person_id,participant_num,tour_id,participant_id +220958279010,2209582790,2223759,5389226,1,220958279,22095827901 +220958279020,2209582790,2223759,5389227,2,220958279,22095827902 +100798519030,1007985190,1173905,2458502,1,100798519,10079851903 +100798519040,1007985190,1173905,2458503,2,100798519,10079851904 +130727777020,1307277770,1402945,3188483,1,130727777,13072777702 +130727777030,1307277770,1402945,3188484,2,130727777,13072777703 +130727777040,1307277770,1402945,3188485,3,130727777,13072777704 diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/override_persons.csv b/activitysim/estimation/test/test_edb_creation/survey_data/override_persons.csv new file mode 100644 index 000000000..030bcafa7 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/override_persons.csv @@ -0,0 +1,91 @@ +person_id,household_id,age,PNUM,sex,pemploy,pstudent,ptype,school_zone_id,workplace_zone_id,free_parking_at_work,cdap_activity,mandatory_tour_frequency,_escort,_shopping,_othmaint,_othdiscr,_eatout,_social,non_mandatory_tour_frequency +26686,26686,39,1,1,3,3,4,-1,-1,False,N,,0,0,1,0,1,0,12 +26844,26844,51,1,1,3,3,4,-1,-1,False,N,,0,0,0,0,0,1,2 +27726,27726,52,1,1,3,3,4,-1,-1,False,N,,0,0,0,1,0,0,1 +29625,29625,61,1,1,3,3,4,-1,-1,False,N,,0,1,0,0,0,0,16 +110675,110675,30,1,1,2,3,2,-1,19,False,H,,0,0,0,0,0,0,0 +112064,112064,48,1,2,2,3,2,-1,21,False,N,,0,1,0,1,0,0,17 +264107,226869,28,1,1,2,3,2,-1,24,False,M,work1,0,0,0,0,0,0,0 +264108,226869,27,2,2,3,3,4,-1,-1,False,N,,0,0,0,1,0,0,1 +323689,256660,22,1,2,2,3,2,-1,1,False,M,work1,0,0,0,0,0,0,0 +323690,256660,23,2,2,2,3,2,-1,5,False,H,,0,0,0,0,0,0,0 +325431,257531,22,1,2,2,3,2,-1,12,False,M,work1,0,0,0,0,0,0,0 +325432,257531,22,2,2,2,3,2,-1,2,False,N,,0,1,0,0,0,0,16 +595684,370497,52,1,1,2,3,2,-1,4,False,H,,0,0,0,0,0,0,0 +595685,370497,18,2,1,3,1,6,13,-1,False,H,,0,0,0,0,0,0,0 +595686,370497,14,3,2,4,1,7,8,-1,False,M,school1,0,0,0,0,0,0,0 +644290,386699,47,1,1,2,3,2,-1,12,False,M,work1,0,0,0,0,0,0,0 +644291,386699,43,2,2,1,3,1,-1,2,False,M,work1,0,0,1,0,0,0,8 +644292,386699,7,3,1,4,1,7,9,-1,False,M,school1,0,0,0,0,0,0,0 +644476,386761,47,1,1,2,3,2,-1,4,False,M,work1,1,0,1,1,0,0,41 +644477,386761,43,2,2,1,3,1,-1,15,False,N,,0,1,0,0,0,0,16 +644478,386761,7,3,1,4,1,7,20,-1,False,M,school1,0,0,0,0,0,0,0 +1265898,568785,80,1,1,3,3,5,-1,-1,False,N,,0,0,0,1,0,0,1 +1267567,570454,85,1,1,3,3,5,-1,-1,False,H,,0,0,0,0,0,0,0 +1427193,703381,65,1,1,3,3,5,-1,-1,False,N,,0,1,0,0,0,0,16 +1427194,703381,69,2,2,3,3,5,-1,-1,False,N,,0,0,1,0,0,1,10 +1572659,763879,60,1,1,3,3,4,-1,-1,False,N,,0,1,0,0,0,0,16 +1572930,764150,52,1,1,3,3,4,-1,-1,False,N,,0,1,1,0,0,0,24 +1632206,823426,31,1,1,1,3,1,-1,2,False,H,,0,0,0,0,0,0,0 +1632281,823501,36,1,1,1,3,1,-1,13,False,M,work1,0,0,0,0,0,0,0 +1632987,824207,29,1,1,2,3,2,-1,4,False,M,work1,0,0,0,0,0,0,0 +1875721,982875,25,1,1,1,3,1,-1,10,False,M,work1,0,0,0,0,0,0,0 +1875722,982875,25,2,2,2,2,3,13,4,False,N,,0,0,0,0,1,0,4 +2159057,1099626,35,1,1,1,3,1,-1,2,False,M,work1,0,0,0,0,0,0,0 +2159058,1099626,36,2,2,3,2,3,9,-1,False,M,school1,0,0,0,0,0,0,0 +2159059,1099626,3,3,1,4,1,8,20,-1,False,M,school1,0,0,0,0,0,0,0 +2458500,1173905,40,1,1,3,3,4,-1,-1,False,N,,0,0,1,0,0,0,8 +2458501,1173905,42,2,2,3,3,4,-1,-1,False,N,,1,0,0,0,0,0,32 +2458502,1173905,12,3,2,4,1,7,8,-1,False,M,school1,0,0,0,0,0,0,0 +2458503,1173905,9,4,1,4,1,7,8,-1,False,M,school1,0,0,0,0,0,0,0 +2566698,1196298,36,1,1,1,3,1,-1,1,False,M,work1,0,0,0,0,0,0,0 +2566699,1196298,29,2,2,3,3,4,-1,-1,False,H,,0,0,0,0,0,0,0 +2566700,1196298,8,3,1,4,1,7,25,-1,False,M,school1,0,0,0,0,0,0,0 +2566701,1196298,4,4,2,4,1,8,3,-1,False,M,school1,0,0,0,0,0,0,0 +2566702,1196298,2,5,1,4,1,8,6,-1,False,M,school1,0,0,0,0,0,0,0 +2936550,1286259,67,1,2,3,3,5,-1,-1,False,N,,0,1,0,1,0,0,17 +2936848,1286557,67,1,1,3,3,5,-1,-1,False,N,,0,0,0,1,1,0,5 +3061894,1363467,68,1,1,3,3,5,-1,-1,False,N,,0,1,0,0,0,0,16 +3061895,1363467,63,2,2,2,3,2,-1,25,False,M,work1,0,0,1,1,0,0,9 +3188482,1402945,66,1,1,2,3,2,-1,24,False,M,work1,0,0,0,0,0,0,0 +3188483,1402945,48,2,2,2,3,2,-1,2,False,M,work1,0,0,0,0,0,0,0 +3188484,1402945,22,3,2,2,2,3,13,2,False,M,work_and_school,0,0,0,0,0,0,0 +3188485,1402945,19,4,2,2,2,3,9,2,False,M,work1,0,0,0,0,0,0,0 +3232955,1444715,42,1,1,2,3,2,-1,8,False,H,,0,0,0,0,0,0,0 +3233462,1445222,43,1,2,1,3,1,-1,22,False,M,work1,0,2,0,1,0,0,17 +3328568,1511234,53,1,2,1,3,1,-1,1,False,M,work1,0,0,0,0,0,0,0 +3328569,1511234,61,2,1,3,2,3,13,-1,False,N,,0,1,0,0,1,0,20 +3495342,1594621,31,1,1,1,3,1,-1,11,False,M,work1,0,0,0,1,0,0,1 +3495343,1594621,25,2,2,2,3,2,-1,9,False,M,work1,0,0,0,0,0,0,0 +3596364,1645132,56,1,2,3,2,3,9,-1,False,N,,0,1,0,0,1,0,20 +3596365,1645132,13,2,2,4,1,7,10,-1,False,N,,0,0,0,0,0,1,2 +3891102,1747467,36,1,2,1,3,1,-1,11,False,M,work1,0,0,0,0,0,0,0 +3891103,1747467,67,2,1,2,3,2,-1,12,False,N,,0,1,0,0,0,0,16 +3891104,1747467,8,3,1,4,1,7,17,-1,False,M,school1,0,0,0,2,0,0,1 +4171615,1810015,29,1,1,3,2,3,12,-1,False,M,school1,0,0,0,0,0,0,0 +4171616,1810015,20,2,1,3,3,4,-1,-1,False,N,,0,0,1,0,0,0,8 +4171617,1810015,27,3,1,1,3,1,-1,15,False,M,work1,0,0,0,0,0,0,0 +4171618,1810015,24,4,1,2,3,2,-1,13,False,H,,0,0,0,0,0,0,0 +4171619,1810015,58,5,2,3,3,4,-1,-1,False,N,,0,1,0,0,0,0,16 +4171620,1810015,3,6,2,4,1,8,8,-1,False,M,school1,0,0,0,0,0,0,0 +4171621,1810015,19,7,2,2,1,6,13,22,False,H,,0,0,0,0,0,0,0 +4171622,1810015,31,8,2,3,3,4,-1,-1,False,N,,0,1,0,0,0,0,16 +4171623,1810015,36,9,2,1,3,1,-1,21,False,M,work1,0,0,0,0,0,0,0 +4823797,1952792,74,1,1,2,3,2,-1,12,False,M,work1,0,0,1,1,0,0,9 +5057160,2048204,40,1,1,1,3,1,-1,23,False,M,work1,0,0,0,0,0,0,0 +5057338,2048382,33,1,1,1,3,1,-1,2,False,M,work1,0,1,1,0,1,1,30 +5387762,2223027,49,1,1,1,3,1,-1,4,False,M,work1,0,0,0,0,0,0,0 +5387763,2223027,39,2,2,1,3,1,-1,14,False,M,work1,0,0,0,0,0,0,0 +5389226,2223759,28,1,2,1,3,1,-1,1,False,M,work1,0,0,0,0,0,0,0 +5389227,2223759,29,2,1,1,3,1,-1,23,False,M,work1,0,0,0,0,0,0,0 +7305540,2727273,72,1,1,2,3,2,-1,20,False,M,work1,0,0,0,1,0,0,1 +7305541,2727273,62,2,2,2,3,2,-1,9,False,M,work2,0,0,0,0,0,0,0 +7453413,2762078,57,1,1,3,3,4,-1,-1,False,N,,0,1,0,0,0,0,16 +7511873,2820538,18,1,1,2,3,2,-1,11,False,H,,0,0,0,0,0,0,0 +7512109,2820774,35,1,1,1,3,1,-1,2,False,M,work1,0,0,0,0,0,0,0 +7512514,2821179,60,1,1,2,3,2,-1,22,False,N,,0,1,0,0,1,1,22 +7513432,2822097,39,1,2,2,3,2,-1,7,False,M,work1,0,0,0,0,0,0,0 +7513554,2822219,18,1,2,1,3,1,-1,9,False,M,work1,0,0,0,0,0,0,0 +7513565,2822230,19,1,2,2,2,3,12,25,False,M,work1,0,0,0,0,0,0,0 +7523517,2832182,87,1,2,3,3,5,-1,-1,False,N,,0,0,0,1,0,0,1 +7523764,2832429,93,1,2,3,3,5,-1,-1,False,H,,0,0,0,0,0,0,0 diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/override_tours.csv b/activitysim/estimation/test/test_edb_creation/survey_data/override_tours.csv new file mode 100644 index 000000000..be63b500f --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/override_tours.csv @@ -0,0 +1,118 @@ +tour_id,survey_tour_id,person_id,household_id,tour_type,tour_category,destination,origin,start,end,tour_mode,survey_parent_tour_id,parent_tour_id,composition,tdd,atwork_subtour_frequency,stop_frequency +10828426,108284260,264107,226869,work,mandatory,24,9,7,19,WALK_LRF,,,,49,no_subtours,0out_0in +13271288,132712880,323689,256660,work,mandatory,1,10,11,18,WALK_LRF,,,,106,no_subtours,0out_0in +13342710,133427100,325431,257531,work,mandatory,12,16,7,17,TNC_SINGLE,,,,47,no_subtours,1out_3in +24423157,244231570,595686,370497,school,mandatory,8,21,7,16,WALK_LOC,,,,46,,0out_0in +26415929,264159290,644290,386699,work,mandatory,12,7,12,22,WALK_LOC,,,,122,no_subtours,0out_0in +26415970,264159700,644291,386699,work,mandatory,2,7,7,17,WALK,,,,47,no_subtours,0out_0in +26416003,264160030,644292,386699,school,mandatory,9,7,13,19,WALK_LRF,,,,130,,0out_1in +26423555,264235550,644476,386761,work,mandatory,4,16,7,17,SHARED3FREE,,,,47,no_subtours,0out_0in +26423629,264236290,644478,386761,school,mandatory,20,16,11,19,WALK_LRF,,,,107,,0out_0in +66923560,669235600,1632281,823501,work,mandatory,13,12,7,19,TNC_SINGLE,,,,49,eat,0out_0in +66952506,669525060,1632987,824207,work,mandatory,4,18,15,21,WALK_LRF,,,,151,no_subtours,0out_0in +76904600,769046000,1875721,982875,work,mandatory,10,16,8,18,WALK_LOC,,,,64,no_subtours,0out_0in +88521376,885213760,2159057,1099626,work,mandatory,2,20,7,18,WALK_LRF,,,,48,no_subtours,0out_1in +88521409,885214090,2159058,1099626,school,mandatory,9,20,15,18,TAXI,,,,148,,0out_0in +88521450,885214500,2159059,1099626,school,mandatory,20,20,8,13,WALK,,,,59,,0out_0in +100798613,1007986130,2458502,1173905,school,mandatory,8,8,8,18,WALK,,,,64,,1out_0in +100798654,1007986540,2458503,1173905,school,mandatory,8,8,8,14,WALK,,,,60,,0out_0in +105234657,1052346570,2566698,1196298,work,mandatory,1,25,6,17,TNC_SINGLE,,,,30,no_subtours,0out_0in +105234731,1052347310,2566700,1196298,school,mandatory,25,25,7,15,WALK,,,,45,,0out_0in +105234772,1052347720,2566701,1196298,school,mandatory,3,25,8,13,WALK,,,,59,,0out_0in +105234813,1052348130,2566702,1196298,school,mandatory,6,25,12,20,WALK_LOC,,,,120,,0out_1in +125537734,1255377340,3061895,1363467,work,mandatory,25,24,6,13,WALK,,,,26,no_subtours,0out_0in +130727801,1307278010,3188482,1402945,work,mandatory,24,25,8,17,WALK,,,,63,no_subtours,1out_0in +130727842,1307278420,3188483,1402945,work,mandatory,2,25,9,17,WALK_LOC,,,,78,no_subtours,0out_0in +130727883,1307278830,3188484,1402945,work,mandatory,2,25,9,12,WALK_LOC,,,,73,no_subtours,0out_0in +130727875,1307278750,3188484,1402945,school,mandatory,13,25,16,17,WALK,,,,155,,0out_2in +130727924,1307279240,3188485,1402945,work,mandatory,2,25,10,21,BIKE,,,,96,no_subtours,1out_1in +132571981,1325719810,3233462,1445222,work,mandatory,22,17,7,23,TNC_SINGLE,,,,53,no_subtours,0out_0in +136471327,1364713270,3328568,1511234,work,mandatory,1,8,6,17,WALK_LRF,,,,30,no_subtours,0out_0in +143309061,1433090610,3495342,1594621,work,mandatory,11,10,7,16,WALK_LOC,,,,46,no_subtours,0out_0in +143309102,1433091020,3495343,1594621,work,mandatory,9,10,13,17,WALK,,,,128,eat,0out_0in +159535221,1595352210,3891102,1747467,work,mandatory,11,16,7,16,WALK_LOC,,,,46,no_subtours,0out_0in +159535295,1595352950,3891104,1747467,school,mandatory,17,16,7,14,WALK_LRF,,,,44,,0out_0in +171036246,1710362460,4171615,1810015,school,mandatory,12,16,13,13,WALK,,,,124,,0out_0in +171036336,1710363360,4171617,1810015,work,mandatory,15,16,9,18,WALK,,,,79,no_subtours,1out_0in +171036451,1710364510,4171620,1810015,school,mandatory,8,16,7,15,WALK_LOC,,,,45,,0out_0in +171036582,1710365820,4171623,1810015,work,mandatory,21,16,8,23,WALK,,,,69,eat,1out_1in +197775716,1977757160,4823797,1952792,work,mandatory,12,14,11,15,BIKE,,,,103,no_subtours,0out_0in +207343599,2073435990,5057160,2048204,work,mandatory,23,5,7,13,TNC_SINGLE,,,,43,no_subtours,0out_0in +207350897,2073508970,5057338,2048382,work,mandatory,2,7,6,17,TNC_SINGLE,,,,30,no_subtours,0out_0in +220898281,2208982810,5387762,2223027,work,mandatory,4,9,7,17,WALK_HVY,,,,47,no_subtours,0out_0in +220898322,2208983220,5387763,2223027,work,mandatory,14,9,8,17,WALK_HVY,,,,63,no_subtours,1out_1in +220958305,2209583050,5389226,2223759,work,mandatory,1,16,9,18,WALK,,,,79,eat,0out_0in +220958346,2209583460,5389227,2223759,work,mandatory,23,16,6,20,WALK,,,,33,no_subtours,0out_0in +299527179,2995271790,7305540,2727273,work,mandatory,20,20,11,15,DRIVEALONEFREE,,,,103,no_subtours,0out_0in +299527220,2995272200,7305541,2727273,work,mandatory,9,20,6,13,WALK,,,,26,no_subtours,0out_0in +299527221,2995272210,7305541,2727273,work,mandatory,9,20,14,18,WALK,,,,139,no_subtours,0out_0in +307996508,3079965080,7512109,2820774,work,mandatory,2,8,8,19,WALK,,,,65,eat,0out_2in +308050751,3080507510,7513432,2822097,work,mandatory,7,8,5,15,BIKE,,,,10,no_subtours,0out_0in +308055753,3080557530,7513554,2822219,work,mandatory,9,8,5,16,WALK,,,,11,no_subtours,0out_0in +308056204,3080562040,7513565,2822230,work,mandatory,25,8,9,18,WALK,,,,79,no_subtours,1out_0in +220958279,2209582790,5389226,2223759,eatout,joint,13,16,20,20,WALK,,,adults,180,,0out_0in +100798519,1007985190,2458502,1173905,shopping,joint,5,8,8,8,SHARED2FREE,,,children,54,,0out_0in +130727777,1307277770,3188483,1402945,othdiscr,joint,24,25,9,9,WALK,,,adults,70,,0out_0in +1094154,10941540,26686,26686,othmaint,non_mandatory,9,8,12,13,BIKE,,,,113,,0out_0in +1094132,10941320,26686,26686,eatout,non_mandatory,5,8,19,19,WALK,,,,175,,0out_0in +1100640,11006400,26844,26844,social,non_mandatory,7,8,8,12,WALK,,,,58,,0out_1in +1136791,11367910,27726,27726,othdiscr,non_mandatory,9,10,8,10,WALK,,,,56,,0out_0in +1214658,12146580,29625,29625,shopping,non_mandatory,17,17,15,15,WALK,,,,145,,0out_0in +4594657,45946570,112064,112064,shopping,non_mandatory,3,16,16,17,WALK,,,,155,,0out_0in +4594649,45946490,112064,112064,othdiscr,non_mandatory,21,16,12,14,DRIVEALONEFREE,,,,114,,0out_0in +10828453,108284530,264108,226869,othdiscr,non_mandatory,19,9,10,11,WALK,,,,86,,0out_0in +13342745,133427450,325432,257531,shopping,non_mandatory,14,16,12,13,TAXI,,,,113,,0out_0in +26415959,264159590,644291,386699,othmaint,non_mandatory,2,7,19,23,TNC_SINGLE,,,,179,,0out_0in +26423525,264235250,644476,386761,escort,non_mandatory,11,16,5,6,TNC_SINGLE,,,,1,,0out_0in +26423544,264235440,644476,386761,othmaint,non_mandatory,13,16,18,19,WALK,,,,170,,0out_0in +26423541,264235410,644476,386761,othdiscr,non_mandatory,16,16,18,18,WALK,,,,169,,0out_0in +26423590,264235900,644477,386761,shopping,non_mandatory,16,16,14,14,DRIVEALONEFREE,,,,135,,0out_0in +51901843,519018430,1265898,568785,othdiscr,non_mandatory,20,17,9,20,WALK_LRF,,,,81,,0out_0in +58514946,585149460,1427193,703381,shopping,non_mandatory,14,25,13,19,WALK,,,,130,,0out_0in +58514982,585149820,1427194,703381,othmaint,non_mandatory,14,25,7,13,WALK,,,,43,,0out_0in +58514990,585149900,1427194,703381,social,non_mandatory,2,25,16,22,WALK,,,,160,,1out_3in +64479052,644790520,1572659,763879,shopping,non_mandatory,24,6,7,20,WALK,,,,50,,2out_2in +64490163,644901630,1572930,764150,shopping,non_mandatory,7,9,11,16,WALK,,,,104,,0out_0in +64490158,644901580,1572930,764150,othmaint,non_mandatory,9,9,8,11,WALK,,,,57,,1out_0in +76904608,769046080,1875722,982875,eatout,non_mandatory,14,16,10,17,WALK,,,,92,,1out_0in +100798528,1007985280,2458500,1173905,othmaint,non_mandatory,7,8,12,20,WALK,,,,120,,0out_0in +100798550,1007985500,2458501,1173905,escort,non_mandatory,16,8,15,16,WALK_LOC,,,,146,,0out_0in +120398583,1203985830,2936550,1286259,shopping,non_mandatory,11,8,12,13,WALK,,,,113,,0out_0in +120398575,1203985750,2936550,1286259,othdiscr,non_mandatory,6,8,14,17,WALK,,,,138,,0out_0in +120410793,1204107930,2936848,1286557,othdiscr,non_mandatory,11,11,15,16,DRIVEALONEFREE,,,,146,,1out_0in +120410774,1204107740,2936848,1286557,eatout,non_mandatory,9,11,14,15,DRIVEALONEFREE,,,,136,,0out_0in +125537687,1255376870,3061894,1363467,shopping,non_mandatory,20,24,12,13,DRIVEALONEFREE,,,,113,,0out_0in +125537723,1255377230,3061895,1363467,othmaint,non_mandatory,7,24,15,16,WALK,,,,146,,0out_0in +125537720,1255377200,3061895,1363467,othdiscr,non_mandatory,9,24,17,19,WALK_HVY,,,,164,,0out_0in +132571975,1325719750,3233462,1445222,shopping,non_mandatory,16,17,6,6,WALK_LRF,,,,19,,0out_0in +132571976,1325719760,3233462,1445222,shopping,non_mandatory,13,17,7,7,WALK_LOC,,,,37,,0out_0in +132571967,1325719670,3233462,1445222,othdiscr,non_mandatory,14,17,7,7,WALK_LRF,,,,37,,0out_0in +136471362,1364713620,3328569,1511234,shopping,non_mandatory,4,8,10,10,WALK_LRF,,,,85,,1out_0in +136471335,1364713350,3328569,1511234,eatout,non_mandatory,13,8,14,16,WALK_LRF,,,,137,,0out_0in +143309047,1433090470,3495342,1594621,othdiscr,non_mandatory,15,10,16,22,TNC_SINGLE,,,,160,,0out_0in +147450957,1474509570,3596364,1645132,shopping,non_mandatory,22,9,19,20,WALK_LRF,,,,176,,0out_2in +147450930,1474509300,3596364,1645132,eatout,non_mandatory,5,9,14,14,WALK,,,,135,,0out_0in +147451001,1474510010,3596365,1645132,social,non_mandatory,9,9,7,18,TAXI,,,,48,,0out_0in +159535256,1595352560,3891103,1747467,shopping,non_mandatory,16,16,13,16,WALK,,,,127,,0out_0in +159535289,1595352890,3891104,1747467,othdiscr,non_mandatory,2,16,14,15,WALK,,,,136,,0out_0in +159535290,1595352900,3891104,1747467,othdiscr,non_mandatory,19,16,16,21,WALK_LOC,,,,159,,0out_0in +171036284,1710362840,4171616,1810015,othmaint,non_mandatory,3,16,9,11,WALK,,,,72,,0out_0in +171036412,1710364120,4171619,1810015,shopping,non_mandatory,1,16,13,15,WALK,,,,126,,0out_1in +171036535,1710365350,4171622,1810015,shopping,non_mandatory,19,16,9,15,WALK,,,,76,,0out_0in +197775705,1977757050,4823797,1952792,othmaint,non_mandatory,7,14,15,15,DRIVEALONEFREE,,,,145,,0out_0in +197775702,1977757020,4823797,1952792,othdiscr,non_mandatory,16,14,16,18,WALK,,,,156,,0out_0in +207350891,2073508910,5057338,2048382,shopping,non_mandatory,25,7,21,21,WALK_LOC,,,,184,,0out_0in +207350886,2073508860,5057338,2048382,othmaint,non_mandatory,7,7,18,18,WALK,,,,169,,0out_0in +207350864,2073508640,5057338,2048382,eatout,non_mandatory,10,7,20,20,WALK_LRF,,,,180,,0out_0in +207350894,2073508940,5057338,2048382,social,non_mandatory,6,7,18,19,TNC_SINGLE,,,,170,,0out_0in +299527165,2995271650,7305540,2727273,othdiscr,non_mandatory,12,20,15,21,WALK,,,,151,,0out_0in +305589966,3055899660,7453413,2762078,shopping,non_mandatory,19,20,16,16,WALK,,,,154,,0out_0in +308013107,3080131070,7512514,2821179,shopping,non_mandatory,6,8,17,19,WALK,,,,164,,0out_0in +308013080,3080130800,7512514,2821179,eatout,non_mandatory,13,8,8,16,WALK,,,,62,,0out_0in +308013110,3080131100,7512514,2821179,social,non_mandatory,9,8,16,16,WALK_LOC,,,,154,,0out_0in +308464222,3084642220,7523517,2832182,othdiscr,non_mandatory,9,7,8,13,WALK_LOC,,,,59,,0out_0in +66923525,669235250,1632281,823501,eat,atwork,8,13,13,14,WALK,669235600.0,66923560.0,,125,,1out_0in +143309067,1433090670,3495343,1594621,eat,atwork,16,9,14,14,WALK,1433091020.0,143309102.0,,135,,3out_0in +171036547,1710365470,4171623,1810015,eat,atwork,7,21,10,10,WALK,1710365820.0,171036582.0,,85,,0out_1in +220958270,2209582700,5389226,2223759,eat,atwork,2,1,13,13,WALK,2209583050.0,220958305.0,,124,,0out_0in +307996473,3079964730,7512109,2820774,eat,atwork,8,2,10,10,WALK,3079965080.0,307996508.0,,85,,0out_1in diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/override_trips.csv b/activitysim/estimation/test/test_edb_creation/survey_data/override_trips.csv new file mode 100644 index 000000000..e2f36dab7 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/override_trips.csv @@ -0,0 +1,278 @@ +trip_id,survey_trip_id,person_id,household_id,survey_tour_id,outbound,purpose,destination,origin,depart,trip_mode,tour_id,trip_num +8753057,87530570,26686,26686,10941320,True,eatout,5,8,19,WALK,1094132,1 +8753061,87530610,26686,26686,10941320,False,home,8,5,19,WALK,1094132,1 +8753233,87532330,26686,26686,10941540,True,othmaint,9,8,12,BIKE,1094154,1 +8753237,87532370,26686,26686,10941540,False,home,8,9,13,WALK,1094154,1 +8805121,88051210,26844,26844,11006400,True,social,7,8,8,WALK,1100640,1 +8805125,88051250,26844,26844,11006400,False,othmaint,7,7,12,WALK,1100640,1 +8805126,88051260,26844,26844,11006400,False,home,8,7,12,WALK,1100640,2 +9094329,90943290,27726,27726,11367910,True,othdiscr,9,10,8,WALK,1136791,1 +9094333,90943330,27726,27726,11367910,False,home,10,9,10,WALK,1136791,1 +9717265,97172650,29625,29625,12146580,True,shopping,17,17,15,WALK,1214658,1 +9717269,97172690,29625,29625,12146580,False,home,17,17,15,WALK,1214658,1 +36757193,367571930,112064,112064,45946490,True,othdiscr,21,16,12,WALK,4594649,1 +36757197,367571970,112064,112064,45946490,False,home,16,21,14,TAXI,4594649,1 +36757257,367572570,112064,112064,45946570,True,shopping,3,16,16,WALK,4594657,1 +36757261,367572610,112064,112064,45946570,False,home,16,3,17,WALK,4594657,1 +86627409,866274090,264107,226869,108284260,True,work,24,9,7,WALK_LRF,10828426,1 +86627413,866274130,264107,226869,108284260,False,home,9,24,19,WALK_LRF,10828426,1 +86627625,866276250,264108,226869,108284530,True,othdiscr,19,9,10,WALK,10828453,1 +86627629,866276290,264108,226869,108284530,False,home,9,19,11,WALK,10828453,1 +106170305,1061703050,323689,256660,132712880,True,work,1,10,11,WALK_LRF,13271288,1 +106170309,1061703090,323689,256660,132712880,False,home,10,1,18,WALK_LRF,13271288,1 +106741681,1067416810,325431,257531,133427100,True,shopping,25,16,7,WALK,13342710,1 +106741682,1067416820,325431,257531,133427100,True,work,12,25,8,WALK,13342710,2 +106741685,1067416850,325431,257531,133427100,False,eatout,7,12,17,WALK,13342710,1 +106741686,1067416860,325431,257531,133427100,False,shopping,25,7,17,WALK_LOC,13342710,2 +106741687,1067416870,325431,257531,133427100,False,escort,7,25,17,WALK_LOC,13342710,3 +106741688,1067416880,325431,257531,133427100,False,home,16,7,17,WALK,13342710,4 +106741961,1067419610,325432,257531,133427450,True,shopping,14,16,12,WALK_LOC,13342745,1 +106741965,1067419650,325432,257531,133427450,False,home,16,14,13,WALK_LOC,13342745,1 +195385257,1953852570,595686,370497,244231570,True,school,8,21,7,WALK_LOC,24423157,1 +195385261,1953852610,595686,370497,244231570,False,home,21,8,16,WALK,24423157,1 +211327433,2113274330,644290,386699,264159290,True,work,12,7,12,WALK_LOC,26415929,1 +211327437,2113274370,644290,386699,264159290,False,home,7,12,22,WALK_LOC,26415929,1 +211327673,2113276730,644291,386699,264159590,True,othmaint,2,7,19,WALK_LOC,26415959,1 +211327677,2113276770,644291,386699,264159590,False,home,7,2,23,TNC_SINGLE,26415959,1 +211327761,2113277610,644291,386699,264159700,True,work,2,7,7,WALK,26415970,1 +211327765,2113277650,644291,386699,264159700,False,home,7,2,17,WALK,26415970,1 +211328025,2113280250,644292,386699,264160030,True,school,9,7,13,WALK_LOC,26416003,1 +211328029,2113280290,644292,386699,264160030,False,othdiscr,7,9,19,WALK_LRF,26416003,1 +211328030,2113280300,644292,386699,264160030,False,home,7,7,19,WALK,26416003,2 +211388201,2113882010,644476,386761,264235250,True,escort,11,16,5,WALK_LOC,26423525,1 +211388205,2113882050,644476,386761,264235250,False,home,16,11,6,WALK_LOC,26423525,1 +211388329,2113883290,644476,386761,264235410,True,othdiscr,16,16,18,WALK,26423541,1 +211388333,2113883330,644476,386761,264235410,False,home,16,16,18,WALK,26423541,1 +211388353,2113883530,644476,386761,264235440,True,othmaint,13,16,18,WALK,26423544,1 +211388357,2113883570,644476,386761,264235440,False,home,16,13,19,WALK,26423544,1 +211388441,2113884410,644476,386761,264235550,True,work,4,16,7,SHARED2FREE,26423555,1 +211388445,2113884450,644476,386761,264235550,False,home,16,4,17,SHARED3FREE,26423555,1 +211388721,2113887210,644477,386761,264235900,True,shopping,16,16,14,WALK,26423590,1 +211388725,2113887250,644477,386761,264235900,False,home,16,16,14,WALK,26423590,1 +211389033,2113890330,644478,386761,264236290,True,school,20,16,11,WALK_LOC,26423629,1 +211389037,2113890370,644478,386761,264236290,False,home,16,20,19,WALK_LRF,26423629,1 +415214745,4152147450,1265898,568785,519018430,True,othdiscr,20,17,9,WALK_LRF,51901843,1 +415214749,4152147490,1265898,568785,519018430,False,home,17,20,20,WALK_LRF,51901843,1 +468119569,4681195690,1427193,703381,585149460,True,shopping,14,25,13,WALK,58514946,1 +468119573,4681195730,1427193,703381,585149460,False,home,25,14,19,WALK,58514946,1 +468119857,4681198570,1427194,703381,585149820,True,othmaint,14,25,7,WALK,58514982,1 +468119861,4681198610,1427194,703381,585149820,False,home,25,14,13,WALK,58514982,1 +468119921,4681199210,1427194,703381,585149900,True,eatout,6,25,16,WALK,58514990,1 +468119922,4681199220,1427194,703381,585149900,True,social,2,6,19,WALK,58514990,2 +468119925,4681199250,1427194,703381,585149900,False,social,6,2,21,WALK,58514990,1 +468119926,4681199260,1427194,703381,585149900,False,shopping,8,6,22,WALK,58514990,2 +468119927,4681199270,1427194,703381,585149900,False,social,7,8,22,WALK,58514990,3 +468119928,4681199280,1427194,703381,585149900,False,home,25,7,22,WALK,58514990,4 +515832417,5158324170,1572659,763879,644790520,True,othmaint,25,6,7,WALK,64479052,1 +515832418,5158324180,1572659,763879,644790520,True,escort,25,25,12,WALK,64479052,2 +515832419,5158324190,1572659,763879,644790520,True,shopping,24,25,17,WALK,64479052,3 +515832421,5158324210,1572659,763879,644790520,False,shopping,25,24,18,WALK,64479052,1 +515832422,5158324220,1572659,763879,644790520,False,escort,7,25,20,WALK,64479052,2 +515832423,5158324230,1572659,763879,644790520,False,home,6,7,20,WALK,64479052,3 +515921265,5159212650,1572930,764150,644901580,True,othmaint,7,9,8,WALK,64490158,1 +515921266,5159212660,1572930,764150,644901580,True,othmaint,9,7,8,WALK,64490158,2 +515921269,5159212690,1572930,764150,644901580,False,home,9,9,11,WALK,64490158,1 +515921305,5159213050,1572930,764150,644901630,True,shopping,7,9,11,WALK,64490163,1 +515921309,5159213090,1572930,764150,644901630,False,home,9,7,16,WALK,64490163,1 +535388201,5353882010,1632281,823501,669235250,True,work,9,13,13,WALK,66923525,1 +535388202,5353882020,1632281,823501,669235250,True,atwork,8,9,13,WALK,66923525,2 +535388205,5353882050,1632281,823501,669235250,False,work,13,8,14,WALK,66923525,1 +535388481,5353884810,1632281,823501,669235600,True,work,13,12,7,WALK,66923560,1 +535388485,5353884850,1632281,823501,669235600,False,home,12,13,19,WALK,66923560,1 +535620049,5356200490,1632987,824207,669525060,True,work,4,18,15,WALK_LOC,66952506,1 +535620053,5356200530,1632987,824207,669525060,False,home,18,4,21,WALK,66952506,1 +615236801,6152368010,1875721,982875,769046000,True,work,10,16,8,WALK_LOC,76904600,1 +615236805,6152368050,1875721,982875,769046000,False,home,16,10,18,WALK_LOC,76904600,1 +615236865,6152368650,1875722,982875,769046080,True,escort,7,16,10,WALK,76904608,1 +615236866,6152368660,1875722,982875,769046080,True,eatout,14,7,13,WALK,76904608,2 +615236869,6152368690,1875722,982875,769046080,False,home,16,14,17,WALK,76904608,1 +708171009,7081710090,2159057,1099626,885213760,True,work,2,20,7,WALK,88521376,1 +708171013,7081710130,2159057,1099626,885213760,False,shopping,8,2,18,WALK,88521376,1 +708171014,7081710140,2159057,1099626,885213760,False,home,20,8,18,WALK_LOC,88521376,2 +708171273,7081712730,2159058,1099626,885214090,True,univ,9,20,15,WALK_LOC,88521409,1 +708171277,7081712770,2159058,1099626,885214090,False,home,20,9,18,WALK_LOC,88521409,1 +708171601,7081716010,2159059,1099626,885214500,True,school,20,20,8,WALK,88521450,1 +708171605,7081716050,2159059,1099626,885214500,False,home,20,20,13,WALK,88521450,1 +806388153,8063881530,2458502,1173905,1007985190,True,shopping,5,8,8,SHARED2FREE,100798519,1 +806388157,8063881570,2458502,1173905,1007985190,False,home,8,5,8,SHARED2FREE,100798519,1 +806388225,8063882250,2458500,1173905,1007985280,True,othmaint,7,8,12,WALK,100798528,1 +806388229,8063882290,2458500,1173905,1007985280,False,home,8,7,20,WALK,100798528,1 +806388401,8063884010,2458501,1173905,1007985500,True,escort,16,8,15,WALK_LOC,100798550,1 +806388405,8063884050,2458501,1173905,1007985500,False,home,8,16,16,WALK_LOC,100798550,1 +806388905,8063889050,2458502,1173905,1007986130,True,social,7,8,8,WALK,100798613,1 +806388906,8063889060,2458502,1173905,1007986130,True,school,8,7,8,WALK,100798613,2 +806388909,8063889090,2458502,1173905,1007986130,False,home,8,8,18,WALK,100798613,1 +806389233,8063892330,2458503,1173905,1007986540,True,school,8,8,8,WALK,100798654,1 +806389237,8063892370,2458503,1173905,1007986540,False,home,8,8,14,WALK,100798654,1 +841877257,8418772570,2566698,1196298,1052346570,True,work,1,25,6,WALK,105234657,1 +841877261,8418772610,2566698,1196298,1052346570,False,home,25,1,17,WALK_LOC,105234657,1 +841877849,8418778490,2566700,1196298,1052347310,True,school,25,25,7,WALK,105234731,1 +841877853,8418778530,2566700,1196298,1052347310,False,home,25,25,15,WALK,105234731,1 +841878177,8418781770,2566701,1196298,1052347720,True,school,3,25,8,WALK,105234772,1 +841878181,8418781810,2566701,1196298,1052347720,False,home,25,3,13,WALK,105234772,1 +841878505,8418785050,2566702,1196298,1052348130,True,school,6,25,12,WALK_LOC,105234813,1 +841878509,8418785090,2566702,1196298,1052348130,False,shopping,25,6,20,WALK_LOC,105234813,1 +841878510,8418785100,2566702,1196298,1052348130,False,home,25,25,20,WALK,105234813,2 +963188601,9631886010,2936550,1286259,1203985750,True,othdiscr,6,8,14,WALK,120398575,1 +963188605,9631886050,2936550,1286259,1203985750,False,home,8,6,17,WALK,120398575,1 +963188665,9631886650,2936550,1286259,1203985830,True,shopping,11,8,12,WALK,120398583,1 +963188669,9631886690,2936550,1286259,1203985830,False,home,8,11,13,WALK,120398583,1 +963286193,9632861930,2936848,1286557,1204107740,True,eatout,9,11,14,WALK,120410774,1 +963286197,9632861970,2936848,1286557,1204107740,False,home,11,9,15,WALK,120410774,1 +963286345,9632863450,2936848,1286557,1204107930,True,escort,10,11,15,WALK,120410793,1 +963286346,9632863460,2936848,1286557,1204107930,True,othdiscr,11,10,16,WALK,120410793,2 +963286349,9632863490,2936848,1286557,1204107930,False,home,11,11,16,WALK,120410793,1 +1004301497,10043014970,3061894,1363467,1255376870,True,shopping,20,24,12,TNC_SHARED,125537687,1 +1004301501,10043015010,3061894,1363467,1255376870,False,home,24,20,13,DRIVEALONEFREE,125537687,1 +1004301761,10043017610,3061895,1363467,1255377200,True,othdiscr,9,24,17,WALK_HVY,125537720,1 +1004301765,10043017650,3061895,1363467,1255377200,False,home,24,9,19,WALK_LRF,125537720,1 +1004301785,10043017850,3061895,1363467,1255377230,True,othmaint,7,24,15,WALK,125537723,1 +1004301789,10043017890,3061895,1363467,1255377230,False,home,24,7,16,WALK,125537723,1 +1004301873,10043018730,3061895,1363467,1255377340,True,work,25,24,6,WALK,125537734,1 +1004301877,10043018770,3061895,1363467,1255377340,False,home,24,25,13,WALK,125537734,1 +1045822217,10458222170,3188483,1402945,1307277770,True,othdiscr,24,25,9,WALK,130727777,1 +1045822221,10458222210,3188483,1402945,1307277770,False,home,25,24,9,WALK,130727777,1 +1045822409,10458224090,3188482,1402945,1307278010,True,work,7,25,8,WALK,130727801,1 +1045822410,10458224100,3188482,1402945,1307278010,True,work,24,7,9,WALK,130727801,2 +1045822413,10458224130,3188482,1402945,1307278010,False,home,25,24,17,WALK,130727801,1 +1045822737,10458227370,3188483,1402945,1307278420,True,work,2,25,9,WALK,130727842,1 +1045822741,10458227410,3188483,1402945,1307278420,False,home,25,2,17,WALK_LOC,130727842,1 +1045823001,10458230010,3188484,1402945,1307278750,True,univ,13,25,16,WALK,130727875,1 +1045823005,10458230050,3188484,1402945,1307278750,False,shopping,7,13,17,WALK,130727875,1 +1045823006,10458230060,3188484,1402945,1307278750,False,othmaint,7,7,17,WALK,130727875,2 +1045823007,10458230070,3188484,1402945,1307278750,False,home,25,7,17,WALK,130727875,3 +1045823065,10458230650,3188484,1402945,1307278830,True,work,2,25,9,WALK,130727883,1 +1045823069,10458230690,3188484,1402945,1307278830,False,home,25,2,12,WALK_LOC,130727883,1 +1045823393,10458233930,3188485,1402945,1307279240,True,escort,25,25,10,BIKE,130727924,1 +1045823394,10458233940,3188485,1402945,1307279240,True,work,2,25,12,BIKE,130727924,2 +1045823397,10458233970,3188485,1402945,1307279240,False,eatout,7,2,20,BIKE,130727924,1 +1045823398,10458233980,3188485,1402945,1307279240,False,home,25,7,21,BIKE,130727924,2 +1060575737,10605757370,3233462,1445222,1325719670,True,othdiscr,14,17,7,WALK,132571967,1 +1060575741,10605757410,3233462,1445222,1325719670,False,home,17,14,7,WALK_LOC,132571967,1 +1060575801,10605758010,3233462,1445222,1325719750,True,shopping,16,17,6,WALK,132571975,1 +1060575805,10605758050,3233462,1445222,1325719750,False,home,17,16,6,WALK_LRF,132571975,1 +1060575809,10605758090,3233462,1445222,1325719760,True,shopping,13,17,7,WALK_LOC,132571976,1 +1060575813,10605758130,3233462,1445222,1325719760,False,home,17,13,7,WALK_LOC,132571976,1 +1060575849,10605758490,3233462,1445222,1325719810,True,work,22,17,7,WALK_LRF,132571981,1 +1060575853,10605758530,3233462,1445222,1325719810,False,home,17,22,23,TNC_SINGLE,132571981,1 +1091770617,10917706170,3328568,1511234,1364713270,True,work,1,8,6,WALK_LRF,136471327,1 +1091770621,10917706210,3328568,1511234,1364713270,False,home,8,1,17,WALK,136471327,1 +1091770681,10917706810,3328569,1511234,1364713350,True,eatout,13,8,14,WALK,136471335,1 +1091770685,10917706850,3328569,1511234,1364713350,False,home,8,13,16,WALK_LRF,136471335,1 +1091770897,10917708970,3328569,1511234,1364713620,True,eatout,9,8,10,WALK_LOC,136471362,1 +1091770898,10917708980,3328569,1511234,1364713620,True,shopping,4,9,10,WALK_LRF,136471362,2 +1091770901,10917709010,3328569,1511234,1364713620,False,home,8,4,10,WALK_LRF,136471362,1 +1146472377,11464723770,3495342,1594621,1433090470,True,othdiscr,15,10,16,WALK_LOC,143309047,1 +1146472381,11464723810,3495342,1594621,1433090470,False,home,10,15,22,WALK_LRF,143309047,1 +1146472489,11464724890,3495342,1594621,1433090610,True,work,11,10,7,WALK_LOC,143309061,1 +1146472493,11464724930,3495342,1594621,1433090610,False,home,10,11,16,WALK,143309061,1 +1146472537,11464725370,3495343,1594621,1433090670,True,othmaint,6,9,14,WALK,143309067,1 +1146472538,11464725380,3495343,1594621,1433090670,True,shopping,6,6,14,WALK,143309067,2 +1146472539,11464725390,3495343,1594621,1433090670,True,escort,7,6,14,WALK,143309067,3 +1146472540,11464725400,3495343,1594621,1433090670,True,atwork,16,7,14,WALK,143309067,4 +1146472541,11464725410,3495343,1594621,1433090670,False,work,9,16,14,WALK,143309067,1 +1146472817,11464728170,3495343,1594621,1433091020,True,work,9,10,13,WALK,143309102,1 +1146472821,11464728210,3495343,1594621,1433091020,False,home,10,9,17,WALK,143309102,1 +1179607441,11796074410,3596364,1645132,1474509300,True,eatout,5,9,14,WALK,147450930,1 +1179607445,11796074450,3596364,1645132,1474509300,False,home,9,5,14,WALK,147450930,1 +1179607657,11796076570,3596364,1645132,1474509570,True,shopping,22,9,19,WALK_LRF,147450957,1 +1179607661,11796076610,3596364,1645132,1474509570,False,shopping,8,22,20,WALK_LRF,147450957,1 +1179607662,11796076620,3596364,1645132,1474509570,False,eatout,9,8,20,WALK_LOC,147450957,2 +1179607663,11796076630,3596364,1645132,1474509570,False,home,9,9,20,WALK,147450957,3 +1179608009,11796080090,3596365,1645132,1474510010,True,social,9,9,7,WALK,147451001,1 +1179608013,11796080130,3596365,1645132,1474510010,False,home,9,9,18,WALK,147451001,1 +1276281769,12762817690,3891102,1747467,1595352210,True,work,11,16,7,WALK_LOC,159535221,1 +1276281773,12762817730,3891102,1747467,1595352210,False,home,16,11,16,WALK_LOC,159535221,1 +1276282049,12762820490,3891103,1747467,1595352560,True,shopping,16,16,13,WALK,159535256,1 +1276282053,12762820530,3891103,1747467,1595352560,False,home,16,16,16,WALK,159535256,1 +1276282313,12762823130,3891104,1747467,1595352890,True,othdiscr,2,16,14,WALK,159535289,1 +1276282317,12762823170,3891104,1747467,1595352890,False,home,16,2,15,WALK,159535289,1 +1276282321,12762823210,3891104,1747467,1595352900,True,othdiscr,19,16,16,WALK_LOC,159535290,1 +1276282325,12762823250,3891104,1747467,1595352900,False,home,16,19,21,WALK_LOC,159535290,1 +1276282361,12762823610,3891104,1747467,1595352950,True,school,17,16,7,WALK_LRF,159535295,1 +1276282365,12762823650,3891104,1747467,1595352950,False,home,16,17,14,WALK_LRF,159535295,1 +1368289969,13682899690,4171615,1810015,1710362460,True,univ,12,16,13,WALK,171036246,1 +1368289973,13682899730,4171615,1810015,1710362460,False,home,16,12,13,WALK,171036246,1 +1368290273,13682902730,4171616,1810015,1710362840,True,othmaint,3,16,9,WALK,171036284,1 +1368290277,13682902770,4171616,1810015,1710362840,False,home,16,3,11,WALK,171036284,1 +1368290689,13682906890,4171617,1810015,1710363360,True,social,8,16,9,WALK,171036336,1 +1368290690,13682906900,4171617,1810015,1710363360,True,work,15,8,12,WALK,171036336,2 +1368290693,13682906930,4171617,1810015,1710363360,False,home,16,15,18,WALK,171036336,1 +1368291297,13682912970,4171619,1810015,1710364120,True,shopping,1,16,13,WALK,171036412,1 +1368291301,13682913010,4171619,1810015,1710364120,False,shopping,7,1,14,WALK,171036412,1 +1368291302,13682913020,4171619,1810015,1710364120,False,home,16,7,15,WALK,171036412,2 +1368291609,13682916090,4171620,1810015,1710364510,True,school,8,16,7,WALK_LOC,171036451,1 +1368291613,13682916130,4171620,1810015,1710364510,False,home,16,8,15,WALK_LOC,171036451,1 +1368292281,13682922810,4171622,1810015,1710365350,True,shopping,19,16,9,WALK,171036535,1 +1368292285,13682922850,4171622,1810015,1710365350,False,home,16,19,15,WALK,171036535,1 +1368292377,13682923770,4171623,1810015,1710365470,True,atwork,7,21,10,WALK,171036547,1 +1368292381,13682923810,4171623,1810015,1710365470,False,othmaint,6,7,10,WALK,171036547,1 +1368292382,13682923820,4171623,1810015,1710365470,False,work,21,6,10,WALK,171036547,2 +1368292657,13682926570,4171623,1810015,1710365820,True,escort,25,16,8,WALK,171036582,1 +1368292658,13682926580,4171623,1810015,1710365820,True,work,21,25,8,WALK,171036582,2 +1368292661,13682926610,4171623,1810015,1710365820,False,work,7,21,23,WALK,171036582,1 +1368292662,13682926620,4171623,1810015,1710365820,False,home,16,7,23,WALK,171036582,2 +1582205617,15822056170,4823797,1952792,1977757020,True,othdiscr,16,14,16,WALK,197775702,1 +1582205621,15822056210,4823797,1952792,1977757020,False,home,14,16,18,WALK,197775702,1 +1582205641,15822056410,4823797,1952792,1977757050,True,othmaint,7,14,15,WALK,197775705,1 +1582205645,15822056450,4823797,1952792,1977757050,False,home,14,7,15,WALK,197775705,1 +1582205729,15822057290,4823797,1952792,1977757160,True,work,12,14,11,BIKE,197775716,1 +1582205733,15822057330,4823797,1952792,1977757160,False,home,14,12,15,BIKE,197775716,1 +1658748793,16587487930,5057160,2048204,2073435990,True,work,23,5,7,WALK_LRF,207343599,1 +1658748797,16587487970,5057160,2048204,2073435990,False,home,5,23,13,WALK_LOC,207343599,1 +1658806913,16588069130,5057338,2048382,2073508640,True,eatout,10,7,20,WALK,207350864,1 +1658806917,16588069170,5057338,2048382,2073508640,False,home,7,10,20,WALK,207350864,1 +1658807089,16588070890,5057338,2048382,2073508860,True,othmaint,7,7,18,WALK,207350886,1 +1658807093,16588070930,5057338,2048382,2073508860,False,home,7,7,18,WALK,207350886,1 +1658807129,16588071290,5057338,2048382,2073508910,True,shopping,25,7,21,WALK,207350891,1 +1658807133,16588071330,5057338,2048382,2073508910,False,home,7,25,21,WALK,207350891,1 +1658807153,16588071530,5057338,2048382,2073508940,True,social,6,7,18,WALK_LOC,207350894,1 +1658807157,16588071570,5057338,2048382,2073508940,False,home,7,6,19,WALK_LOC,207350894,1 +1658807177,16588071770,5057338,2048382,2073508970,True,work,2,7,6,WALK,207350897,1 +1658807181,16588071810,5057338,2048382,2073508970,False,home,7,2,17,WALK_LOC,207350897,1 +1767186249,17671862490,5387762,2223027,2208982810,True,work,4,9,7,WALK_HVY,220898281,1 +1767186253,17671862530,5387762,2223027,2208982810,False,home,9,4,17,WALK_LRF,220898281,1 +1767186577,17671865770,5387763,2223027,2208983220,True,escort,7,9,8,WALK,220898322,1 +1767186578,17671865780,5387763,2223027,2208983220,True,work,14,7,8,WALK_LOC,220898322,2 +1767186581,17671865810,5387763,2223027,2208983220,False,othmaint,9,14,17,WALK_LRF,220898322,1 +1767186582,17671865820,5387763,2223027,2208983220,False,home,9,9,17,WALK,220898322,2 +1767666161,17676661610,5389226,2223759,2209582700,True,atwork,2,1,13,WALK,220958270,1 +1767666165,17676661650,5389226,2223759,2209582700,False,work,1,2,13,WALK,220958270,1 +1767666233,17676662330,5389226,2223759,2209582790,True,eatout,13,16,20,TNC_SINGLE,220958279,1 +1767666237,17676662370,5389226,2223759,2209582790,False,home,16,13,20,TAXI,220958279,1 +1767666441,17676664410,5389226,2223759,2209583050,True,work,1,16,9,WALK,220958305,1 +1767666445,17676664450,5389226,2223759,2209583050,False,home,16,1,18,WALK,220958305,1 +1767666769,17676667690,5389227,2223759,2209583460,True,work,23,16,6,WALK,220958346,1 +1767666773,17676667730,5389227,2223759,2209583460,False,home,16,23,20,WALK,220958346,1 +2396217321,23962173210,7305540,2727273,2995271650,True,othdiscr,12,20,15,WALK,299527165,1 +2396217325,23962173250,7305540,2727273,2995271650,False,home,20,12,21,WALK,299527165,1 +2396217433,23962174330,7305540,2727273,2995271790,True,work,20,20,11,WALK,299527179,1 +2396217437,23962174370,7305540,2727273,2995271790,False,home,20,20,15,WALK,299527179,1 +2396217761,23962177610,7305541,2727273,2995272200,True,work,9,20,6,WALK,299527220,1 +2396217765,23962177650,7305541,2727273,2995272200,False,home,20,9,13,WALK,299527220,1 +2396217769,23962177690,7305541,2727273,2995272210,True,work,9,20,14,WALK,299527221,1 +2396217773,23962177730,7305541,2727273,2995272210,False,home,20,9,18,WALK,299527221,1 +2444719729,24447197290,7453413,2762078,3055899660,True,shopping,19,20,16,WALK,305589966,1 +2444719733,24447197330,7453413,2762078,3055899660,False,home,20,19,16,WALK,305589966,1 +2463971785,24639717850,7512109,2820774,3079964730,True,atwork,8,2,10,WALK,307996473,1 +2463971789,24639717890,7512109,2820774,3079964730,False,work,7,8,10,WALK,307996473,1 +2463971790,24639717900,7512109,2820774,3079964730,False,work,2,7,10,WALK,307996473,2 +2463972065,24639720650,7512109,2820774,3079965080,True,work,2,8,8,WALK,307996508,1 +2463972069,24639720690,7512109,2820774,3079965080,False,shopping,8,2,18,WALK,307996508,1 +2463972070,24639720700,7512109,2820774,3079965080,False,shopping,8,8,19,WALK,307996508,2 +2463972071,24639720710,7512109,2820774,3079965080,False,home,8,8,19,WALK,307996508,3 +2464104641,24641046410,7512514,2821179,3080130800,True,eatout,13,8,8,WALK,308013080,1 +2464104645,24641046450,7512514,2821179,3080130800,False,home,8,13,16,WALK,308013080,1 +2464104857,24641048570,7512514,2821179,3080131070,True,shopping,6,8,17,WALK,308013107,1 +2464104861,24641048610,7512514,2821179,3080131070,False,home,8,6,19,WALK,308013107,1 +2464104881,24641048810,7512514,2821179,3080131100,True,social,9,8,16,WALK,308013110,1 +2464104885,24641048850,7512514,2821179,3080131100,False,home,8,9,16,WALK_LOC,308013110,1 +2464406009,24644060090,7513432,2822097,3080507510,True,work,7,8,5,BIKE,308050751,1 +2464406013,24644060130,7513432,2822097,3080507510,False,home,8,7,15,BIKE,308050751,1 +2464446025,24644460250,7513554,2822219,3080557530,True,work,9,8,5,WALK,308055753,1 +2464446029,24644460290,7513554,2822219,3080557530,False,home,8,9,16,WALK,308055753,1 +2464449633,24644496330,7513565,2822230,3080562040,True,univ,9,8,9,WALK,308056204,1 +2464449634,24644496340,7513565,2822230,3080562040,True,work,25,9,9,WALK,308056204,2 +2464449637,24644496370,7513565,2822230,3080562040,False,home,8,25,18,WALK,308056204,1 +2467713777,24677137770,7523517,2832182,3084642220,True,othdiscr,9,7,8,WALK,308464222,1 +2467713781,24677137810,7523517,2832182,3084642220,False,home,7,9,13,WALK,308464222,1 diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/survey_households.csv b/activitysim/estimation/test/test_edb_creation/survey_data/survey_households.csv new file mode 100644 index 000000000..77ae6411d --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/survey_households.csv @@ -0,0 +1,51 @@ +household_id,home_zone_id,income,hhsize,HHT,auto_ownership,num_workers +982875,16,30900,2,5,1,2 +1810015,16,99700,9,2,1,4 +1099626,20,58160,3,1,1,1 +763879,6,59220,1,4,1,0 +824207,18,51000,1,4,0,1 +2822230,8,0,1,0,0,1 +2821179,8,0,1,0,0,1 +1196298,25,31360,5,1,0,1 +1363467,24,58300,2,2,1,1 +386761,16,21000,3,1,1,2 +2223027,9,133000,2,1,0,2 +2832182,7,0,1,0,0,0 +2727273,20,118800,2,1,1,2 +1444715,14,60000,1,4,0,1 +112064,16,18000,1,6,1,1 +1952792,14,61900,1,4,1,1 +2223759,16,144100,2,1,0,2 +2820538,8,0,1,0,0,1 +27726,10,24000,1,4,0,0 +570454,21,1500,1,4,0,0 +370497,21,18000,3,2,0,1 +1173905,8,45000,4,1,1,0 +1286557,11,30000,1,4,1,0 +2762078,20,0,1,0,0,0 +2832429,16,0,1,0,1,0 +386699,7,21000,3,1,0,2 +2822097,8,0,1,0,1,1 +764150,9,34630,1,4,1,0 +2820774,8,0,1,0,1,1 +1594621,10,69200,2,1,0,2 +257531,16,12000,2,7,0,2 +1645132,9,92700,2,3,0,0 +29625,17,4200,1,4,0,0 +1402945,25,37200,4,1,1,4 +823501,12,30000,1,4,0,1 +1747467,16,76000,3,1,0,2 +1445222,17,68000,1,6,0,1 +26844,8,3500,1,4,0,0 +2822219,8,0,1,0,0,1 +110675,16,5050,1,4,0,1 +2048204,5,388000,1,4,0,1 +1286259,8,34400,1,6,0,0 +568785,17,8000,1,4,1,0 +26686,8,0,1,4,0,0 +1511234,8,88600,2,1,0,1 +2048382,7,195000,1,4,0,1 +256660,10,27800,2,7,0,2 +703381,25,17210,2,1,1,0 +226869,9,4000,2,1,0,1 +823426,11,50000,1,4,1,1 diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/survey_joint_tour_participants.csv b/activitysim/estimation/test/test_edb_creation/survey_data/survey_joint_tour_participants.csv new file mode 100644 index 000000000..c7c2f6128 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/survey_joint_tour_participants.csv @@ -0,0 +1,8 @@ +participant_id,tour_id,household_id,person_id,participant_num +22095827901,220958279,2223759,5389226,1 +22095827902,220958279,2223759,5389227,2 +10079851903,100798519,1173905,2458502,1 +10079851904,100798519,1173905,2458503,2 +13072777702,130727777,1402945,3188483,1 +13072777703,130727777,1402945,3188484,2 +13072777704,130727777,1402945,3188485,3 diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/survey_persons.csv b/activitysim/estimation/test/test_edb_creation/survey_data/survey_persons.csv new file mode 100644 index 000000000..e2f1c5c62 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/survey_persons.csv @@ -0,0 +1,91 @@ +person_id,household_id,age,PNUM,sex,pemploy,pstudent,ptype,school_zone_id,workplace_zone_id,free_parking_at_work +26686,26686,39,1,1,3,3,4,-1,-1,False +26844,26844,51,1,1,3,3,4,-1,-1,False +27726,27726,52,1,1,3,3,4,-1,-1,False +29625,29625,61,1,1,3,3,4,-1,-1,False +110675,110675,30,1,1,2,3,2,-1,19,False +112064,112064,48,1,2,2,3,2,-1,21,False +264107,226869,28,1,1,2,3,2,-1,24,False +264108,226869,27,2,2,3,3,4,-1,-1,False +323689,256660,22,1,2,2,3,2,-1,1,False +323690,256660,23,2,2,2,3,2,-1,5,False +325431,257531,22,1,2,2,3,2,-1,12,False +325432,257531,22,2,2,2,3,2,-1,2,False +595684,370497,52,1,1,2,3,2,-1,4,False +595685,370497,18,2,1,3,1,6,13,-1,False +595686,370497,14,3,2,4,1,7,8,-1,False +644290,386699,47,1,1,2,3,2,-1,12,False +644291,386699,43,2,2,1,3,1,-1,2,False +644292,386699,7,3,1,4,1,7,9,-1,False +644476,386761,47,1,1,2,3,2,-1,4,False +644477,386761,43,2,2,1,3,1,-1,15,False +644478,386761,7,3,1,4,1,7,20,-1,False +1265898,568785,80,1,1,3,3,5,-1,-1,False +1267567,570454,85,1,1,3,3,5,-1,-1,False +1427193,703381,65,1,1,3,3,5,-1,-1,False +1427194,703381,69,2,2,3,3,5,-1,-1,False +1572659,763879,60,1,1,3,3,4,-1,-1,False +1572930,764150,52,1,1,3,3,4,-1,-1,False +1632206,823426,31,1,1,1,3,1,-1,2,False +1632281,823501,36,1,1,1,3,1,-1,13,False +1632987,824207,29,1,1,2,3,2,-1,4,False +1875721,982875,25,1,1,1,3,1,-1,10,False +1875722,982875,25,2,2,2,2,3,13,4,False +2159057,1099626,35,1,1,1,3,1,-1,2,False +2159058,1099626,36,2,2,3,2,3,9,-1,False +2159059,1099626,3,3,1,4,1,8,20,-1,False +2458500,1173905,40,1,1,3,3,4,-1,-1,False +2458501,1173905,42,2,2,3,3,4,-1,-1,False +2458502,1173905,12,3,2,4,1,7,8,-1,False +2458503,1173905,9,4,1,4,1,7,8,-1,False +2566698,1196298,36,1,1,1,3,1,-1,1,False +2566699,1196298,29,2,2,3,3,4,-1,-1,False +2566700,1196298,8,3,1,4,1,7,25,-1,False +2566701,1196298,4,4,2,4,1,8,3,-1,False +2566702,1196298,2,5,1,4,1,8,6,-1,False +2936550,1286259,67,1,2,3,3,5,-1,-1,False +2936848,1286557,67,1,1,3,3,5,-1,-1,False +3061894,1363467,68,1,1,3,3,5,-1,-1,False +3061895,1363467,63,2,2,2,3,2,-1,25,False +3188482,1402945,66,1,1,2,3,2,-1,24,False +3188483,1402945,48,2,2,2,3,2,-1,2,False +3188484,1402945,22,3,2,2,2,3,13,2,False +3188485,1402945,19,4,2,2,2,3,9,2,False +3232955,1444715,42,1,1,2,3,2,-1,8,False +3233462,1445222,43,1,2,1,3,1,-1,22,False +3328568,1511234,53,1,2,1,3,1,-1,1,False +3328569,1511234,61,2,1,3,2,3,13,-1,False +3495342,1594621,31,1,1,1,3,1,-1,11,False +3495343,1594621,25,2,2,2,3,2,-1,9,False +3596364,1645132,56,1,2,3,2,3,9,-1,False +3596365,1645132,13,2,2,4,1,7,10,-1,False +3891102,1747467,36,1,2,1,3,1,-1,11,False +3891103,1747467,67,2,1,2,3,2,-1,12,False +3891104,1747467,8,3,1,4,1,7,17,-1,False +4171615,1810015,29,1,1,3,2,3,12,-1,False +4171616,1810015,20,2,1,3,3,4,-1,-1,False +4171617,1810015,27,3,1,1,3,1,-1,15,False +4171618,1810015,24,4,1,2,3,2,-1,13,False +4171619,1810015,58,5,2,3,3,4,-1,-1,False +4171620,1810015,3,6,2,4,1,8,8,-1,False +4171621,1810015,19,7,2,2,1,6,13,22,False +4171622,1810015,31,8,2,3,3,4,-1,-1,False +4171623,1810015,36,9,2,1,3,1,-1,21,False +4823797,1952792,74,1,1,2,3,2,-1,12,False +5057160,2048204,40,1,1,1,3,1,-1,23,False +5057338,2048382,33,1,1,1,3,1,-1,2,False +5387762,2223027,49,1,1,1,3,1,-1,4,False +5387763,2223027,39,2,2,1,3,1,-1,14,False +5389226,2223759,28,1,2,1,3,1,-1,1,False +5389227,2223759,29,2,1,1,3,1,-1,23,False +7305540,2727273,72,1,1,2,3,2,-1,20,False +7305541,2727273,62,2,2,2,3,2,-1,9,False +7453413,2762078,57,1,1,3,3,4,-1,-1,False +7511873,2820538,18,1,1,2,3,2,-1,11,False +7512109,2820774,35,1,1,1,3,1,-1,2,False +7512514,2821179,60,1,1,2,3,2,-1,22,False +7513432,2822097,39,1,2,2,3,2,-1,7,False +7513554,2822219,18,1,2,1,3,1,-1,9,False +7513565,2822230,19,1,2,2,2,3,12,25,False +7523517,2832182,87,1,2,3,3,5,-1,-1,False +7523764,2832429,93,1,2,3,3,5,-1,-1,False diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/survey_tours.csv b/activitysim/estimation/test/test_edb_creation/survey_data/survey_tours.csv new file mode 100644 index 000000000..02a2a170f --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/survey_tours.csv @@ -0,0 +1,118 @@ +tour_id,person_id,household_id,tour_type,tour_category,destination,origin,start,end,tour_mode,parent_tour_id +10828426,264107,226869,work,mandatory,24,9,7,19,WALK_LRF, +13271288,323689,256660,work,mandatory,1,10,11,18,WALK_LRF, +13342710,325431,257531,work,mandatory,12,16,7,17,TNC_SINGLE, +24423157,595686,370497,school,mandatory,8,21,7,16,WALK_LOC, +26415929,644290,386699,work,mandatory,12,7,12,22,WALK_LOC, +26415970,644291,386699,work,mandatory,2,7,7,17,WALK, +26416003,644292,386699,school,mandatory,9,7,13,19,WALK_LRF, +26423555,644476,386761,work,mandatory,4,16,7,17,SHARED3FREE, +26423629,644478,386761,school,mandatory,20,16,11,19,WALK_LRF, +66923560,1632281,823501,work,mandatory,13,12,7,19,TNC_SINGLE, +66952506,1632987,824207,work,mandatory,4,18,15,21,WALK_LRF, +76904600,1875721,982875,work,mandatory,10,16,8,18,WALK_LOC, +88521376,2159057,1099626,work,mandatory,2,20,7,18,WALK_LRF, +88521409,2159058,1099626,school,mandatory,9,20,15,18,TAXI, +88521450,2159059,1099626,school,mandatory,20,20,8,13,WALK, +100798613,2458502,1173905,school,mandatory,8,8,8,18,WALK, +100798654,2458503,1173905,school,mandatory,8,8,8,14,WALK, +105234657,2566698,1196298,work,mandatory,1,25,6,17,TNC_SINGLE, +105234731,2566700,1196298,school,mandatory,25,25,7,15,WALK, +105234772,2566701,1196298,school,mandatory,3,25,8,13,WALK, +105234813,2566702,1196298,school,mandatory,6,25,12,20,WALK_LOC, +125537734,3061895,1363467,work,mandatory,25,24,6,13,WALK, +130727801,3188482,1402945,work,mandatory,24,25,8,17,WALK, +130727842,3188483,1402945,work,mandatory,2,25,9,17,WALK_LOC, +130727883,3188484,1402945,work,mandatory,2,25,9,12,WALK_LOC, +130727875,3188484,1402945,school,mandatory,13,25,16,17,WALK, +130727924,3188485,1402945,work,mandatory,2,25,10,21,BIKE, +132571981,3233462,1445222,work,mandatory,22,17,7,23,TNC_SINGLE, +136471327,3328568,1511234,work,mandatory,1,8,6,17,WALK_LRF, +143309061,3495342,1594621,work,mandatory,11,10,7,16,WALK_LOC, +143309102,3495343,1594621,work,mandatory,9,10,13,17,WALK, +159535221,3891102,1747467,work,mandatory,11,16,7,16,WALK_LOC, +159535295,3891104,1747467,school,mandatory,17,16,7,14,WALK_LRF, +171036246,4171615,1810015,school,mandatory,12,16,13,13,WALK, +171036336,4171617,1810015,work,mandatory,15,16,9,18,WALK, +171036451,4171620,1810015,school,mandatory,8,16,7,15,WALK_LOC, +171036582,4171623,1810015,work,mandatory,21,16,8,23,WALK, +197775716,4823797,1952792,work,mandatory,12,14,11,15,BIKE, +207343599,5057160,2048204,work,mandatory,23,5,7,13,TNC_SINGLE, +207350897,5057338,2048382,work,mandatory,2,7,6,17,TNC_SINGLE, +220898281,5387762,2223027,work,mandatory,4,9,7,17,WALK_HVY, +220898322,5387763,2223027,work,mandatory,14,9,8,17,WALK_HVY, +220958305,5389226,2223759,work,mandatory,1,16,9,18,WALK, +220958346,5389227,2223759,work,mandatory,23,16,6,20,WALK, +299527179,7305540,2727273,work,mandatory,20,20,11,15,DRIVEALONEFREE, +299527220,7305541,2727273,work,mandatory,9,20,6,13,WALK, +299527221,7305541,2727273,work,mandatory,9,20,14,18,WALK, +307996508,7512109,2820774,work,mandatory,2,8,8,19,WALK, +308050751,7513432,2822097,work,mandatory,7,8,5,15,BIKE, +308055753,7513554,2822219,work,mandatory,9,8,5,16,WALK, +308056204,7513565,2822230,work,mandatory,25,8,9,18,WALK, +220958279,5389226,2223759,eatout,joint,13,16,20,20,WALK, +100798519,2458502,1173905,shopping,joint,5,8,8,8,SHARED2FREE, +130727777,3188483,1402945,othdiscr,joint,24,25,9,9,WALK, +1094154,26686,26686,othmaint,non_mandatory,9,8,12,13,BIKE, +1094132,26686,26686,eatout,non_mandatory,5,8,19,19,WALK, +1100640,26844,26844,social,non_mandatory,7,8,8,12,WALK, +1136791,27726,27726,othdiscr,non_mandatory,9,10,8,10,WALK, +1214658,29625,29625,shopping,non_mandatory,17,17,15,15,WALK, +4594657,112064,112064,shopping,non_mandatory,3,16,16,17,WALK, +4594649,112064,112064,othdiscr,non_mandatory,21,16,12,14,DRIVEALONEFREE, +10828453,264108,226869,othdiscr,non_mandatory,19,9,10,11,WALK, +13342745,325432,257531,shopping,non_mandatory,14,16,12,13,TAXI, +26415959,644291,386699,othmaint,non_mandatory,2,7,19,23,TNC_SINGLE, +26423525,644476,386761,escort,non_mandatory,11,16,5,6,TNC_SINGLE, +26423544,644476,386761,othmaint,non_mandatory,13,16,18,19,WALK, +26423541,644476,386761,othdiscr,non_mandatory,16,16,18,18,WALK, +26423590,644477,386761,shopping,non_mandatory,16,16,14,14,DRIVEALONEFREE, +51901843,1265898,568785,othdiscr,non_mandatory,20,17,9,20,WALK_LRF, +58514946,1427193,703381,shopping,non_mandatory,14,25,13,19,WALK, +58514982,1427194,703381,othmaint,non_mandatory,14,25,7,13,WALK, +58514990,1427194,703381,social,non_mandatory,2,25,16,22,WALK, +64479052,1572659,763879,shopping,non_mandatory,24,6,7,20,WALK, +64490163,1572930,764150,shopping,non_mandatory,7,9,11,16,WALK, +64490158,1572930,764150,othmaint,non_mandatory,9,9,8,11,WALK, +76904608,1875722,982875,eatout,non_mandatory,14,16,10,17,WALK, +100798528,2458500,1173905,othmaint,non_mandatory,7,8,12,20,WALK, +100798550,2458501,1173905,escort,non_mandatory,16,8,15,16,WALK_LOC, +120398583,2936550,1286259,shopping,non_mandatory,11,8,12,13,WALK, +120398575,2936550,1286259,othdiscr,non_mandatory,6,8,14,17,WALK, +120410793,2936848,1286557,othdiscr,non_mandatory,11,11,15,16,DRIVEALONEFREE, +120410774,2936848,1286557,eatout,non_mandatory,9,11,14,15,DRIVEALONEFREE, +125537687,3061894,1363467,shopping,non_mandatory,20,24,12,13,DRIVEALONEFREE, +125537723,3061895,1363467,othmaint,non_mandatory,7,24,15,16,WALK, +125537720,3061895,1363467,othdiscr,non_mandatory,9,24,17,19,WALK_HVY, +132571975,3233462,1445222,shopping,non_mandatory,16,17,6,6,WALK_LRF, +132571976,3233462,1445222,shopping,non_mandatory,13,17,7,7,WALK_LOC, +132571967,3233462,1445222,othdiscr,non_mandatory,14,17,7,7,WALK_LRF, +136471362,3328569,1511234,shopping,non_mandatory,4,8,10,10,WALK_LRF, +136471335,3328569,1511234,eatout,non_mandatory,13,8,14,16,WALK_LRF, +143309047,3495342,1594621,othdiscr,non_mandatory,15,10,16,22,TNC_SINGLE, +147450957,3596364,1645132,shopping,non_mandatory,22,9,19,20,WALK_LRF, +147450930,3596364,1645132,eatout,non_mandatory,5,9,14,14,WALK, +147451001,3596365,1645132,social,non_mandatory,9,9,7,18,TAXI, +159535256,3891103,1747467,shopping,non_mandatory,16,16,13,16,WALK, +159535289,3891104,1747467,othdiscr,non_mandatory,2,16,14,15,WALK, +159535290,3891104,1747467,othdiscr,non_mandatory,19,16,16,21,WALK_LOC, +171036284,4171616,1810015,othmaint,non_mandatory,3,16,9,11,WALK, +171036412,4171619,1810015,shopping,non_mandatory,1,16,13,15,WALK, +171036535,4171622,1810015,shopping,non_mandatory,19,16,9,15,WALK, +197775705,4823797,1952792,othmaint,non_mandatory,7,14,15,15,DRIVEALONEFREE, +197775702,4823797,1952792,othdiscr,non_mandatory,16,14,16,18,WALK, +207350891,5057338,2048382,shopping,non_mandatory,25,7,21,21,WALK_LOC, +207350886,5057338,2048382,othmaint,non_mandatory,7,7,18,18,WALK, +207350864,5057338,2048382,eatout,non_mandatory,10,7,20,20,WALK_LRF, +207350894,5057338,2048382,social,non_mandatory,6,7,18,19,TNC_SINGLE, +299527165,7305540,2727273,othdiscr,non_mandatory,12,20,15,21,WALK, +305589966,7453413,2762078,shopping,non_mandatory,19,20,16,16,WALK, +308013107,7512514,2821179,shopping,non_mandatory,6,8,17,19,WALK, +308013080,7512514,2821179,eatout,non_mandatory,13,8,8,16,WALK, +308013110,7512514,2821179,social,non_mandatory,9,8,16,16,WALK_LOC, +308464222,7523517,2832182,othdiscr,non_mandatory,9,7,8,13,WALK_LOC, +66923525,1632281,823501,eat,atwork,8,13,13,14,WALK,66923560.0 +143309067,3495343,1594621,eat,atwork,16,9,14,14,WALK,143309102.0 +171036547,4171623,1810015,eat,atwork,7,21,10,10,WALK,171036582.0 +220958270,5389226,2223759,eat,atwork,2,1,13,13,WALK,220958305.0 +307996473,7512109,2820774,eat,atwork,8,2,10,10,WALK,307996508.0 diff --git a/activitysim/estimation/test/test_edb_creation/survey_data/survey_trips.csv b/activitysim/estimation/test/test_edb_creation/survey_data/survey_trips.csv new file mode 100644 index 000000000..c8b273247 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/survey_data/survey_trips.csv @@ -0,0 +1,278 @@ +trip_id,person_id,household_id,tour_id,outbound,purpose,destination,origin,depart,trip_mode +8753057,26686,26686,1094132,True,eatout,5,8,19,WALK +8753061,26686,26686,1094132,False,home,8,5,19,WALK +8753233,26686,26686,1094154,True,othmaint,9,8,12,BIKE +8753237,26686,26686,1094154,False,home,8,9,13,WALK +8805121,26844,26844,1100640,True,social,7,8,8,WALK +8805125,26844,26844,1100640,False,othmaint,7,7,12,WALK +8805126,26844,26844,1100640,False,home,8,7,12,WALK +9094329,27726,27726,1136791,True,othdiscr,9,10,8,WALK +9094333,27726,27726,1136791,False,home,10,9,10,WALK +9717265,29625,29625,1214658,True,shopping,17,17,15,WALK +9717269,29625,29625,1214658,False,home,17,17,15,WALK +36757193,112064,112064,4594649,True,othdiscr,21,16,12,WALK +36757197,112064,112064,4594649,False,home,16,21,14,TAXI +36757257,112064,112064,4594657,True,shopping,3,16,16,WALK +36757261,112064,112064,4594657,False,home,16,3,17,WALK +86627409,264107,226869,10828426,True,work,24,9,7,WALK_LRF +86627413,264107,226869,10828426,False,home,9,24,19,WALK_LRF +86627625,264108,226869,10828453,True,othdiscr,19,9,10,WALK +86627629,264108,226869,10828453,False,home,9,19,11,WALK +106170305,323689,256660,13271288,True,work,1,10,11,WALK_LRF +106170309,323689,256660,13271288,False,home,10,1,18,WALK_LRF +106741681,325431,257531,13342710,True,shopping,25,16,7,WALK +106741682,325431,257531,13342710,True,work,12,25,8,WALK +106741685,325431,257531,13342710,False,eatout,7,12,17,WALK +106741686,325431,257531,13342710,False,shopping,25,7,17,WALK_LOC +106741687,325431,257531,13342710,False,escort,7,25,17,WALK_LOC +106741688,325431,257531,13342710,False,home,16,7,17,WALK +106741961,325432,257531,13342745,True,shopping,14,16,12,WALK_LOC +106741965,325432,257531,13342745,False,home,16,14,13,WALK_LOC +195385257,595686,370497,24423157,True,school,8,21,7,WALK_LOC +195385261,595686,370497,24423157,False,home,21,8,16,WALK +211327433,644290,386699,26415929,True,work,12,7,12,WALK_LOC +211327437,644290,386699,26415929,False,home,7,12,22,WALK_LOC +211327673,644291,386699,26415959,True,othmaint,2,7,19,WALK_LOC +211327677,644291,386699,26415959,False,home,7,2,23,TNC_SINGLE +211327761,644291,386699,26415970,True,work,2,7,7,WALK +211327765,644291,386699,26415970,False,home,7,2,17,WALK +211328025,644292,386699,26416003,True,school,9,7,13,WALK_LOC +211328029,644292,386699,26416003,False,othdiscr,7,9,19,WALK_LRF +211328030,644292,386699,26416003,False,home,7,7,19,WALK +211388201,644476,386761,26423525,True,escort,11,16,5,WALK_LOC +211388205,644476,386761,26423525,False,home,16,11,6,WALK_LOC +211388329,644476,386761,26423541,True,othdiscr,16,16,18,WALK +211388333,644476,386761,26423541,False,home,16,16,18,WALK +211388353,644476,386761,26423544,True,othmaint,13,16,18,WALK +211388357,644476,386761,26423544,False,home,16,13,19,WALK +211388441,644476,386761,26423555,True,work,4,16,7,SHARED2FREE +211388445,644476,386761,26423555,False,home,16,4,17,SHARED3FREE +211388721,644477,386761,26423590,True,shopping,16,16,14,WALK +211388725,644477,386761,26423590,False,home,16,16,14,WALK +211389033,644478,386761,26423629,True,school,20,16,11,WALK_LOC +211389037,644478,386761,26423629,False,home,16,20,19,WALK_LRF +415214745,1265898,568785,51901843,True,othdiscr,20,17,9,WALK_LRF +415214749,1265898,568785,51901843,False,home,17,20,20,WALK_LRF +468119569,1427193,703381,58514946,True,shopping,14,25,13,WALK +468119573,1427193,703381,58514946,False,home,25,14,19,WALK +468119857,1427194,703381,58514982,True,othmaint,14,25,7,WALK +468119861,1427194,703381,58514982,False,home,25,14,13,WALK +468119921,1427194,703381,58514990,True,eatout,6,25,16,WALK +468119922,1427194,703381,58514990,True,social,2,6,19,WALK +468119925,1427194,703381,58514990,False,social,6,2,21,WALK +468119926,1427194,703381,58514990,False,shopping,8,6,22,WALK +468119927,1427194,703381,58514990,False,social,7,8,22,WALK +468119928,1427194,703381,58514990,False,home,25,7,22,WALK +515832417,1572659,763879,64479052,True,othmaint,25,6,7,WALK +515832418,1572659,763879,64479052,True,escort,25,25,12,WALK +515832419,1572659,763879,64479052,True,shopping,24,25,17,WALK +515832421,1572659,763879,64479052,False,shopping,25,24,18,WALK +515832422,1572659,763879,64479052,False,escort,7,25,20,WALK +515832423,1572659,763879,64479052,False,home,6,7,20,WALK +515921265,1572930,764150,64490158,True,othmaint,7,9,8,WALK +515921266,1572930,764150,64490158,True,othmaint,9,7,8,WALK +515921269,1572930,764150,64490158,False,home,9,9,11,WALK +515921305,1572930,764150,64490163,True,shopping,7,9,11,WALK +515921309,1572930,764150,64490163,False,home,9,7,16,WALK +535388201,1632281,823501,66923525,True,work,9,13,13,WALK +535388202,1632281,823501,66923525,True,atwork,8,9,13,WALK +535388205,1632281,823501,66923525,False,work,13,8,14,WALK +535388481,1632281,823501,66923560,True,work,13,12,7,WALK +535388485,1632281,823501,66923560,False,home,12,13,19,WALK +535620049,1632987,824207,66952506,True,work,4,18,15,WALK_LOC +535620053,1632987,824207,66952506,False,home,18,4,21,WALK +615236801,1875721,982875,76904600,True,work,10,16,8,WALK_LOC +615236805,1875721,982875,76904600,False,home,16,10,18,WALK_LOC +615236865,1875722,982875,76904608,True,escort,7,16,10,WALK +615236866,1875722,982875,76904608,True,eatout,14,7,13,WALK +615236869,1875722,982875,76904608,False,home,16,14,17,WALK +708171009,2159057,1099626,88521376,True,work,2,20,7,WALK +708171013,2159057,1099626,88521376,False,shopping,8,2,18,WALK +708171014,2159057,1099626,88521376,False,home,20,8,18,WALK_LOC +708171273,2159058,1099626,88521409,True,univ,9,20,15,WALK_LOC +708171277,2159058,1099626,88521409,False,home,20,9,18,WALK_LOC +708171601,2159059,1099626,88521450,True,school,20,20,8,WALK +708171605,2159059,1099626,88521450,False,home,20,20,13,WALK +806388153,2458502,1173905,100798519,True,shopping,5,8,8,SHARED2FREE +806388157,2458502,1173905,100798519,False,home,8,5,8,SHARED2FREE +806388225,2458500,1173905,100798528,True,othmaint,7,8,12,WALK +806388229,2458500,1173905,100798528,False,home,8,7,20,WALK +806388401,2458501,1173905,100798550,True,escort,16,8,15,WALK_LOC +806388405,2458501,1173905,100798550,False,home,8,16,16,WALK_LOC +806388905,2458502,1173905,100798613,True,social,7,8,8,WALK +806388906,2458502,1173905,100798613,True,school,8,7,8,WALK +806388909,2458502,1173905,100798613,False,home,8,8,18,WALK +806389233,2458503,1173905,100798654,True,school,8,8,8,WALK +806389237,2458503,1173905,100798654,False,home,8,8,14,WALK +841877257,2566698,1196298,105234657,True,work,1,25,6,WALK +841877261,2566698,1196298,105234657,False,home,25,1,17,WALK_LOC +841877849,2566700,1196298,105234731,True,school,25,25,7,WALK +841877853,2566700,1196298,105234731,False,home,25,25,15,WALK +841878177,2566701,1196298,105234772,True,school,3,25,8,WALK +841878181,2566701,1196298,105234772,False,home,25,3,13,WALK +841878505,2566702,1196298,105234813,True,school,6,25,12,WALK_LOC +841878509,2566702,1196298,105234813,False,shopping,25,6,20,WALK_LOC +841878510,2566702,1196298,105234813,False,home,25,25,20,WALK +963188601,2936550,1286259,120398575,True,othdiscr,6,8,14,WALK +963188605,2936550,1286259,120398575,False,home,8,6,17,WALK +963188665,2936550,1286259,120398583,True,shopping,11,8,12,WALK +963188669,2936550,1286259,120398583,False,home,8,11,13,WALK +963286193,2936848,1286557,120410774,True,eatout,9,11,14,WALK +963286197,2936848,1286557,120410774,False,home,11,9,15,WALK +963286345,2936848,1286557,120410793,True,escort,10,11,15,WALK +963286346,2936848,1286557,120410793,True,othdiscr,11,10,16,WALK +963286349,2936848,1286557,120410793,False,home,11,11,16,WALK +1004301497,3061894,1363467,125537687,True,shopping,20,24,12,TNC_SHARED +1004301501,3061894,1363467,125537687,False,home,24,20,13,DRIVEALONEFREE +1004301761,3061895,1363467,125537720,True,othdiscr,9,24,17,WALK_HVY +1004301765,3061895,1363467,125537720,False,home,24,9,19,WALK_LRF +1004301785,3061895,1363467,125537723,True,othmaint,7,24,15,WALK +1004301789,3061895,1363467,125537723,False,home,24,7,16,WALK +1004301873,3061895,1363467,125537734,True,work,25,24,6,WALK +1004301877,3061895,1363467,125537734,False,home,24,25,13,WALK +1045822217,3188483,1402945,130727777,True,othdiscr,24,25,9,WALK +1045822221,3188483,1402945,130727777,False,home,25,24,9,WALK +1045822409,3188482,1402945,130727801,True,work,7,25,8,WALK +1045822410,3188482,1402945,130727801,True,work,24,7,9,WALK +1045822413,3188482,1402945,130727801,False,home,25,24,17,WALK +1045822737,3188483,1402945,130727842,True,work,2,25,9,WALK +1045822741,3188483,1402945,130727842,False,home,25,2,17,WALK_LOC +1045823001,3188484,1402945,130727875,True,univ,13,25,16,WALK +1045823005,3188484,1402945,130727875,False,shopping,7,13,17,WALK +1045823006,3188484,1402945,130727875,False,othmaint,7,7,17,WALK +1045823007,3188484,1402945,130727875,False,home,25,7,17,WALK +1045823065,3188484,1402945,130727883,True,work,2,25,9,WALK +1045823069,3188484,1402945,130727883,False,home,25,2,12,WALK_LOC +1045823393,3188485,1402945,130727924,True,escort,25,25,10,BIKE +1045823394,3188485,1402945,130727924,True,work,2,25,12,BIKE +1045823397,3188485,1402945,130727924,False,eatout,7,2,20,BIKE +1045823398,3188485,1402945,130727924,False,home,25,7,21,BIKE +1060575737,3233462,1445222,132571967,True,othdiscr,14,17,7,WALK +1060575741,3233462,1445222,132571967,False,home,17,14,7,WALK_LOC +1060575801,3233462,1445222,132571975,True,shopping,16,17,6,WALK +1060575805,3233462,1445222,132571975,False,home,17,16,6,WALK_LRF +1060575809,3233462,1445222,132571976,True,shopping,13,17,7,WALK_LOC +1060575813,3233462,1445222,132571976,False,home,17,13,7,WALK_LOC +1060575849,3233462,1445222,132571981,True,work,22,17,7,WALK_LRF +1060575853,3233462,1445222,132571981,False,home,17,22,23,TNC_SINGLE +1091770617,3328568,1511234,136471327,True,work,1,8,6,WALK_LRF +1091770621,3328568,1511234,136471327,False,home,8,1,17,WALK +1091770681,3328569,1511234,136471335,True,eatout,13,8,14,WALK +1091770685,3328569,1511234,136471335,False,home,8,13,16,WALK_LRF +1091770897,3328569,1511234,136471362,True,eatout,9,8,10,WALK_LOC +1091770898,3328569,1511234,136471362,True,shopping,4,9,10,WALK_LRF +1091770901,3328569,1511234,136471362,False,home,8,4,10,WALK_LRF +1146472377,3495342,1594621,143309047,True,othdiscr,15,10,16,WALK_LOC +1146472381,3495342,1594621,143309047,False,home,10,15,22,WALK_LRF +1146472489,3495342,1594621,143309061,True,work,11,10,7,WALK_LOC +1146472493,3495342,1594621,143309061,False,home,10,11,16,WALK +1146472537,3495343,1594621,143309067,True,othmaint,6,9,14,WALK +1146472538,3495343,1594621,143309067,True,shopping,6,6,14,WALK +1146472539,3495343,1594621,143309067,True,escort,7,6,14,WALK +1146472540,3495343,1594621,143309067,True,atwork,16,7,14,WALK +1146472541,3495343,1594621,143309067,False,work,9,16,14,WALK +1146472817,3495343,1594621,143309102,True,work,9,10,13,WALK +1146472821,3495343,1594621,143309102,False,home,10,9,17,WALK +1179607441,3596364,1645132,147450930,True,eatout,5,9,14,WALK +1179607445,3596364,1645132,147450930,False,home,9,5,14,WALK +1179607657,3596364,1645132,147450957,True,shopping,22,9,19,WALK_LRF +1179607661,3596364,1645132,147450957,False,shopping,8,22,20,WALK_LRF +1179607662,3596364,1645132,147450957,False,eatout,9,8,20,WALK_LOC +1179607663,3596364,1645132,147450957,False,home,9,9,20,WALK +1179608009,3596365,1645132,147451001,True,social,9,9,7,WALK +1179608013,3596365,1645132,147451001,False,home,9,9,18,WALK +1276281769,3891102,1747467,159535221,True,work,11,16,7,WALK_LOC +1276281773,3891102,1747467,159535221,False,home,16,11,16,WALK_LOC +1276282049,3891103,1747467,159535256,True,shopping,16,16,13,WALK +1276282053,3891103,1747467,159535256,False,home,16,16,16,WALK +1276282313,3891104,1747467,159535289,True,othdiscr,2,16,14,WALK +1276282317,3891104,1747467,159535289,False,home,16,2,15,WALK +1276282321,3891104,1747467,159535290,True,othdiscr,19,16,16,WALK_LOC +1276282325,3891104,1747467,159535290,False,home,16,19,21,WALK_LOC +1276282361,3891104,1747467,159535295,True,school,17,16,7,WALK_LRF +1276282365,3891104,1747467,159535295,False,home,16,17,14,WALK_LRF +1368289969,4171615,1810015,171036246,True,univ,12,16,13,WALK +1368289973,4171615,1810015,171036246,False,home,16,12,13,WALK +1368290273,4171616,1810015,171036284,True,othmaint,3,16,9,WALK +1368290277,4171616,1810015,171036284,False,home,16,3,11,WALK +1368290689,4171617,1810015,171036336,True,social,8,16,9,WALK +1368290690,4171617,1810015,171036336,True,work,15,8,12,WALK +1368290693,4171617,1810015,171036336,False,home,16,15,18,WALK +1368291297,4171619,1810015,171036412,True,shopping,1,16,13,WALK +1368291301,4171619,1810015,171036412,False,shopping,7,1,14,WALK +1368291302,4171619,1810015,171036412,False,home,16,7,15,WALK +1368291609,4171620,1810015,171036451,True,school,8,16,7,WALK_LOC +1368291613,4171620,1810015,171036451,False,home,16,8,15,WALK_LOC +1368292281,4171622,1810015,171036535,True,shopping,19,16,9,WALK +1368292285,4171622,1810015,171036535,False,home,16,19,15,WALK +1368292377,4171623,1810015,171036547,True,atwork,7,21,10,WALK +1368292381,4171623,1810015,171036547,False,othmaint,6,7,10,WALK +1368292382,4171623,1810015,171036547,False,work,21,6,10,WALK +1368292657,4171623,1810015,171036582,True,escort,25,16,8,WALK +1368292658,4171623,1810015,171036582,True,work,21,25,8,WALK +1368292661,4171623,1810015,171036582,False,work,7,21,23,WALK +1368292662,4171623,1810015,171036582,False,home,16,7,23,WALK +1582205617,4823797,1952792,197775702,True,othdiscr,16,14,16,WALK +1582205621,4823797,1952792,197775702,False,home,14,16,18,WALK +1582205641,4823797,1952792,197775705,True,othmaint,7,14,15,WALK +1582205645,4823797,1952792,197775705,False,home,14,7,15,WALK +1582205729,4823797,1952792,197775716,True,work,12,14,11,BIKE +1582205733,4823797,1952792,197775716,False,home,14,12,15,BIKE +1658748793,5057160,2048204,207343599,True,work,23,5,7,WALK_LRF +1658748797,5057160,2048204,207343599,False,home,5,23,13,WALK_LOC +1658806913,5057338,2048382,207350864,True,eatout,10,7,20,WALK +1658806917,5057338,2048382,207350864,False,home,7,10,20,WALK +1658807089,5057338,2048382,207350886,True,othmaint,7,7,18,WALK +1658807093,5057338,2048382,207350886,False,home,7,7,18,WALK +1658807129,5057338,2048382,207350891,True,shopping,25,7,21,WALK +1658807133,5057338,2048382,207350891,False,home,7,25,21,WALK +1658807153,5057338,2048382,207350894,True,social,6,7,18,WALK_LOC +1658807157,5057338,2048382,207350894,False,home,7,6,19,WALK_LOC +1658807177,5057338,2048382,207350897,True,work,2,7,6,WALK +1658807181,5057338,2048382,207350897,False,home,7,2,17,WALK_LOC +1767186249,5387762,2223027,220898281,True,work,4,9,7,WALK_HVY +1767186253,5387762,2223027,220898281,False,home,9,4,17,WALK_LRF +1767186577,5387763,2223027,220898322,True,escort,7,9,8,WALK +1767186578,5387763,2223027,220898322,True,work,14,7,8,WALK_LOC +1767186581,5387763,2223027,220898322,False,othmaint,9,14,17,WALK_LRF +1767186582,5387763,2223027,220898322,False,home,9,9,17,WALK +1767666161,5389226,2223759,220958270,True,atwork,2,1,13,WALK +1767666165,5389226,2223759,220958270,False,work,1,2,13,WALK +1767666233,5389226,2223759,220958279,True,eatout,13,16,20,TNC_SINGLE +1767666237,5389226,2223759,220958279,False,home,16,13,20,TAXI +1767666441,5389226,2223759,220958305,True,work,1,16,9,WALK +1767666445,5389226,2223759,220958305,False,home,16,1,18,WALK +1767666769,5389227,2223759,220958346,True,work,23,16,6,WALK +1767666773,5389227,2223759,220958346,False,home,16,23,20,WALK +2396217321,7305540,2727273,299527165,True,othdiscr,12,20,15,WALK +2396217325,7305540,2727273,299527165,False,home,20,12,21,WALK +2396217433,7305540,2727273,299527179,True,work,20,20,11,WALK +2396217437,7305540,2727273,299527179,False,home,20,20,15,WALK +2396217761,7305541,2727273,299527220,True,work,9,20,6,WALK +2396217765,7305541,2727273,299527220,False,home,20,9,13,WALK +2396217769,7305541,2727273,299527221,True,work,9,20,14,WALK +2396217773,7305541,2727273,299527221,False,home,20,9,18,WALK +2444719729,7453413,2762078,305589966,True,shopping,19,20,16,WALK +2444719733,7453413,2762078,305589966,False,home,20,19,16,WALK +2463971785,7512109,2820774,307996473,True,atwork,8,2,10,WALK +2463971789,7512109,2820774,307996473,False,work,7,8,10,WALK +2463971790,7512109,2820774,307996473,False,work,2,7,10,WALK +2463972065,7512109,2820774,307996508,True,work,2,8,8,WALK +2463972069,7512109,2820774,307996508,False,shopping,8,2,18,WALK +2463972070,7512109,2820774,307996508,False,shopping,8,8,19,WALK +2463972071,7512109,2820774,307996508,False,home,8,8,19,WALK +2464104641,7512514,2821179,308013080,True,eatout,13,8,8,WALK +2464104645,7512514,2821179,308013080,False,home,8,13,16,WALK +2464104857,7512514,2821179,308013107,True,shopping,6,8,17,WALK +2464104861,7512514,2821179,308013107,False,home,8,6,19,WALK +2464104881,7512514,2821179,308013110,True,social,9,8,16,WALK +2464104885,7512514,2821179,308013110,False,home,8,9,16,WALK_LOC +2464406009,7513432,2822097,308050751,True,work,7,8,5,BIKE +2464406013,7513432,2822097,308050751,False,home,8,7,15,BIKE +2464446025,7513554,2822219,308055753,True,work,9,8,5,WALK +2464446029,7513554,2822219,308055753,False,home,8,9,16,WALK +2464449633,7513565,2822230,308056204,True,univ,9,8,9,WALK +2464449634,7513565,2822230,308056204,True,work,25,9,9,WALK +2464449637,7513565,2822230,308056204,False,home,8,25,18,WALK +2467713777,7523517,2832182,308464222,True,othdiscr,9,7,8,WALK +2467713781,7523517,2832182,308464222,False,home,7,9,13,WALK diff --git a/activitysim/estimation/test/test_edb_creation/test_edb_formation.py b/activitysim/estimation/test/test_edb_creation/test_edb_formation.py new file mode 100644 index 000000000..81a803647 --- /dev/null +++ b/activitysim/estimation/test/test_edb_creation/test_edb_formation.py @@ -0,0 +1,263 @@ +import os +import pandas as pd +import numpy as np +import pandas.testing as pdt +import yaml + + +configs_estimation_dir = ( + "activitysim/estimation/test/test_edb_creation/configs_estimation" +) +configs_dir = "activitysim/examples/prototype_mtc/configs" +survey_data_dir = "activitysim/estimation/test/test_edb_creation/survey_data" +data_dir = "activitysim/examples/prototype_mtc/data" +base_output_dir = "activitysim/estimation/test/test_edb_creation/outputs" + + +def launch_est_example(output_dir, multiprocess, fileformat): + + # setting multiprocess setting + settings_template_file = os.path.join( + configs_estimation_dir, "settings_template.yaml" + ) + settings_file = os.path.join(configs_estimation_dir, "settings.yaml") + settings = yaml.safe_load(open(settings_template_file)) + assert multiprocess in [True, False] + settings["multiprocess"] = multiprocess + yaml.dump(settings, open(settings_file, "w")) + + # setting fileformat setting + settings_template_file = os.path.join( + configs_estimation_dir, "estimation_template.yaml" + ) + est_settings_file = os.path.join(configs_estimation_dir, "estimation.yaml") + est_settings = yaml.safe_load(open(settings_template_file)) + assert fileformat in ["csv", "parquet", "pkl"] + est_settings["EDB_FILETYPE"] = fileformat + yaml.dump(est_settings, open(est_settings_file, "w")) + + run_cmd = f"activitysim run -c {configs_estimation_dir} -c {configs_dir} -d {survey_data_dir} -d {data_dir} -o {base_output_dir}/{output_dir}" + print( + f"launching with options output_dir={output_dir} multiprocess={multiprocess} fileformat={fileformat}" + ) + print("launching from ", os.getcwd()) + result = os.system(run_cmd) + assert result == 0, "ActivitySim run failed" + + +def read_table(file_name): + if file_name.endswith(".csv"): + return pd.read_csv(file_name, low_memory=False) + elif file_name.endswith(".parquet"): + df = pd.read_parquet(file_name).reset_index(drop=True) + df.columns.name = None + return df + elif file_name.endswith(".pkl"): + df = pd.read_pickle(file_name).reset_index(drop=True) + df.columns = df.columns.astype(str) + df.columns.name = None + return df + else: + raise ValueError(f"Unsupported file format: {file_name}") + + +def process_table(df): + if "variable" == df.columns[1]: + # need to sort variable column + df = df.sort_values(by=[df.columns[0], "variable"]).reset_index(drop=True) + df.columns.name = None + + if "chunk_id" in df.columns: + # remove chunk_id column + df = df.drop(columns=["chunk_id"]) + + return df + + +def find_lowest_level_directories(starting_directory): + lowest_dirs = list() + + for root, dirs, files in os.walk(starting_directory): + if not dirs: + lowest_dirs.append(root) + + return lowest_dirs + + +def try_compare_with_same_dtypes(ser1, ser2, rtol, atol): + try: + concatenated = pd.concat([ser1, ser2]) + common_type = concatenated.dtype + pdt.assert_series_equal( + ser1.astype(common_type), + ser2.astype(common_type), + check_dtype=False, + rtol=rtol, + atol=atol, + ) + except (ValueError, AssertionError) as e: + return False + return True + + +def try_compare_with_numeric(ser1, ser2, rtol, atol): + try: + ser1_num = pd.to_numeric(ser1, errors="coerce") + ser2_num = pd.to_numeric(ser2, errors="coerce") + pdt.assert_series_equal( + ser1_num, ser2_num, check_dtype=False, rtol=rtol, atol=atol + ) + except AssertionError as e: + return False + return True + + +def try_compare_with_strings(ser1, ser2): + try: + pdt.assert_series_equal(ser1.astype(str), ser2.astype(str), check_dtype=False) + except AssertionError as e: + return False + return True + + +def try_compare_with_combination(ser1, ser2, rtol=1e-5, atol=1e-8): + """ + This is necessary because we have columns like this: + [index]: [0, 1, 2, 3, 4, 5, ...] + [left]: [False, False, 1, 6, AM, -1.3037984712857993, EA, ...] + [right]: [False, False, 1, 6, AM, -1.3037984712857994, EA, ...] + (notice the machine precision difference in the float) + """ + # replace annoying string values of bools + ser1 = ser1.replace({"True": True, "False": False}) + ser2 = ser2.replace({"True": True, "False": False}) + # Separate numerical and non-numerical values + ser1_num = pd.to_numeric(ser1, errors="coerce") + ser2_num = pd.to_numeric(ser2, errors="coerce") + ser1_non_num = ser1[ser1_num.isna()] + ser2_non_num = ser2[ser2_num.isna()] + ser1_num = ser1_num.dropna() + ser2_num = ser2_num.dropna() + try: + pdt.assert_series_equal( + ser1_num, ser2_num, check_dtype=False, rtol=rtol, atol=atol + ) + pdt.assert_series_equal(ser1_non_num, ser2_non_num, check_dtype=False) + except AssertionError as e: + return False + return True + + +def compare_dataframes_with_tolerance(df1, df2, rtol=1e-3, atol=1e-3): + dfs_are_same = True + e_msg = "" + try: + pdt.assert_frame_equal(df1, df2, check_dtype=False, rtol=rtol, atol=atol) + return dfs_are_same, e_msg + except AssertionError as e: + print(e) + print("trying to compare columns individually") + for col in df1.columns: + try: + if try_compare_with_same_dtypes(df1[col], df2[col], rtol, atol): + continue + elif try_compare_with_numeric(df1[col], df2[col], rtol, atol): + continue + elif try_compare_with_strings(df1[col], df2[col]): + continue + elif try_compare_with_combination( + df1[col], df2[col], rtol=rtol, atol=atol + ): + continue + else: + dfs_are_same = False + e_msg += f"Column '{col}' failed: {df1[col]} vs {df2[col]}\n" + print(f"Column '{col}' failed:\n {df1[col]} vs {df2[col]}\n") + except Exception as e: + dfs_are_same = False + e_msg = e + + return dfs_are_same, e_msg + + +def regress_EDBs(regress_folder, output_folder, fileformat="csv"): + edb_file = os.path.join(base_output_dir, regress_folder, "estimation_data_bundle") + + edb_dirs = find_lowest_level_directories(edb_file) + + for dir in edb_dirs: + dir_basename = dir.split("estimation_data_bundle")[1][1:] + output_dir = os.path.join( + base_output_dir, output_folder, "estimation_data_bundle", dir_basename + ) + + for file in os.listdir(dir): + if file.endswith(".yaml"): + continue + print(f"Regressing {file}") + + regress_path = os.path.join(dir, file) + output_path = os.path.join(output_dir, file) + + # regress against csv for parquet, but regress against parquet for pkl (faster) + if not os.path.exists(output_path) and (fileformat == "parquet"): + output_path = output_path.replace(".csv", ".parquet") + if not os.path.exists(output_path) and (fileformat == "pkl"): + output_path = output_path.replace(".parquet", ".pkl") + + try: + regress_df = read_table(regress_path) + output_df = read_table(output_path) + except FileNotFoundError as e: + assert False, f"File not found: {e}" + + regress_df = process_table(regress_df) + output_df = process_table(output_df) + + dfs_are_same, e = compare_dataframes_with_tolerance(regress_df, output_df) + if not dfs_are_same: + assert False, f"Regression test failed for {file} with error: {e}" + return + + +def test_generating_sp_csv(): + # first generate original tables + output_dir = "output_single_csv" + launch_est_example(output_dir, False, "csv") + + +def test_sp_parquet(): + # multiprocess = False, fileformat = "parquet" + output_dir = "output_single_parquet" + launch_est_example(output_dir, False, "parquet") + regress_EDBs("output_single_csv", output_dir, "parquet") + + +def test_sp_pkl(): + # multiprocess = False, fileformat = "pkl" + output_dir = "output_single_pkl" + launch_est_example(output_dir, False, "pkl") + regress_EDBs("output_single_parquet", output_dir, "pkl") + + +def test_mp_parquet(): + # multiprocess = True, fileformat = "parquet" + output_dir = "output_multiprocess_parquet" + launch_est_example(output_dir, True, "parquet") + regress_EDBs("output_single_parquet", output_dir, "parquet") + + +# def test_mp_csv(): +# # multiprocess = True, fileformat = "csv" +# output_dir = "output_multiprocess_csv" +# launch_est_example(output_dir, True, "csv") +# regress_EDBs("output_single_csv", output_dir, "csv") + + +if __name__ == "__main__": + + test_generating_sp_csv() + test_sp_parquet() + test_mp_parquet() + # test_mp_csv() + test_sp_pkl() diff --git a/activitysim/examples/example_estimation/scripts/infer.py b/activitysim/examples/example_estimation/scripts/infer.py index 6b6991992..ab9def642 100644 --- a/activitysim/examples/example_estimation/scripts/infer.py +++ b/activitysim/examples/example_estimation/scripts/infer.py @@ -165,6 +165,7 @@ def read_alts(): alts = read_alts() tour_types = list(alts.columns.values) + tour_types.remove("tot_tours") # tour_frequency is index in alts table alts["alt_id"] = alts.index @@ -806,7 +807,7 @@ def infer(state: workflow.State, configs_dir, input_dir, output_dir): assert skip_controls or check_controls("joint_tour_participants", "index") # patch_tour_ids - trips = patch_trip_ids(tours, trips) + trips = patch_trip_ids(state, tours, trips) survey_tables["trips"]["table"] = trips # so we can check_controls assert skip_controls or check_controls("trips", "index") @@ -860,4 +861,13 @@ def infer(state: workflow.State, configs_dir, input_dir, output_dir): if apply_controls: read_tables(input_dir, control_tables) +state = ( + workflow.State() + .initialize_filesystem( + configs_dir=(configs_dir,), + output_dir=output_dir, + data_dir=(data_dir,), + ) + .load_settings() +) infer(state, configs_dir, input_dir, output_dir)