From eacf192a5eb94c269a63e9b0725bff5d4a4cd71c Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:27:40 -0700 Subject: [PATCH 01/17] v0.4.0 --- grid2demand/__init__.py | 2 +- grid2demand/_grid2demand.py | 21 ++++++++++--------- grid2demand/func_lib/gen_agent_demand.py | 4 ++-- grid2demand/func_lib/gen_zone.py | 6 +++--- grid2demand/func_lib/read_node_poi.py | 8 +++---- .../trip_rate_production_attraction.py | 2 +- requirements.txt | 1 + 7 files changed, 23 insertions(+), 21 deletions(-) diff --git a/grid2demand/__init__.py b/grid2demand/__init__.py index 0c50124..32779e3 100644 --- a/grid2demand/__init__.py +++ b/grid2demand/__init__.py @@ -7,7 +7,7 @@ from .utils_lib.pkg_settings import pkg_settings from ._grid2demand import GRID2DEMAND -print('grid2demand, version 0.3.9') +print('grid2demand, version 0.4.0') __all__ = ["read_node", "read_poi", "read_network", diff --git a/grid2demand/_grid2demand.py b/grid2demand/_grid2demand.py index 6287728..0c8963f 100644 --- a/grid2demand/_grid2demand.py +++ b/grid2demand/_grid2demand.py @@ -9,10 +9,8 @@ import pandas as pd from grid2demand.utils_lib.pkg_settings import pkg_settings from grid2demand.utils_lib.net_utils import Node, POI, Zone, Agent -from grid2demand.utils_lib.utils import (get_filenames_from_folder_by_type, - check_required_files_exist, - gen_unique_filename, - path2linux) +from grid2demand.utils_lib.utils import check_required_files_exist + from grid2demand.func_lib.read_node_poi import read_node, read_poi, read_network, read_zone from grid2demand.func_lib.gen_zone import (net2zone, sync_zone_and_node_geometry, @@ -23,6 +21,9 @@ from grid2demand.func_lib.gravity_model import run_gravity_model, calc_zone_production_attraction from grid2demand.func_lib.gen_agent_demand import gen_agent_based_demand +from pyufunc import (path2linux, get_filenames_by_ext, + generate_unique_filename) + class GRID2DEMAND: @@ -60,7 +61,7 @@ def __check_input_dir(self) -> None: raise NotADirectoryError(f"Error: Input directory {self.input_dir} does not exist.") # check required files in input directory - dir_files = get_filenames_from_folder_by_type(self.input_dir, "csv") + dir_files = get_filenames_by_ext(self.input_dir, "csv") required_files = pkg_settings.get("required_files", []) is_required_files_exist = check_required_files_exist(required_files, dir_files) if not is_required_files_exist: @@ -382,7 +383,7 @@ def save_demand(self) -> None: print("Error: df_demand does not exist. Please run run_gravity_model() first.") return - path_output = gen_unique_filename(path2linux(os.path.join(self.output_dir, "demand.csv"))) + path_output = generate_unique_filename(path2linux(os.path.join(self.output_dir, "demand.csv"))) df_demand_non_zero = self.df_demand[self.df_demand["volume"] > 0] df_demand_non_zero.to_csv(path_output, index=False) print(f" : Successfully saved demand.csv to {self.output_dir}") @@ -394,7 +395,7 @@ def save_agent(self) -> None: print("Error: df_agent does not exist. Please run gen_agent_based_demand() first.") return - path_output = gen_unique_filename(path2linux(os.path.join(self.output_dir, "agent.csv"))) + path_output = generate_unique_filename(path2linux(os.path.join(self.output_dir, "agent.csv"))) self.df_agent.to_csv(path_output, index=False) print(f" : Successfully saved agent.csv to {self.output_dir}") @@ -405,7 +406,7 @@ def save_zone(self) -> None: print("Error: zone_dict does not exist. Please run sync_geometry_between_zone_and_node_poi() first.") return - path_output = gen_unique_filename(path2linux(os.path.join(self.output_dir, "zone.csv"))) + path_output = generate_unique_filename(path2linux(os.path.join(self.output_dir, "zone.csv"))) zone_df = pd.DataFrame(self.zone_dict.values()) zone_df.to_csv(path_output, index=False) print(f" : Successfully saved zone.csv to {self.output_dir}") @@ -417,7 +418,7 @@ def save_zone_od_dist_table(self) -> None: print("Error: zone_od_dist_matrix does not exist. Please run calc_zone_od_distance_matrix() first.") return - path_output = gen_unique_filename(path2linux(os.path.join(self.output_dir, "zone_od_dist_table.csv"))) + path_output = generate_unique_filename(path2linux(os.path.join(self.output_dir, "zone_od_dist_table.csv"))) zone_od_dist_table_df = pd.DataFrame(self.zone_od_dist_matrix.values()) zone_od_dist_table_df = zone_od_dist_table_df[["o_zone_id", "o_zone_name", "d_zone_id", "d_zone_name", "dist_km", "geometry"]] zone_od_dist_table_df.to_csv(path_output, index=False) @@ -430,7 +431,7 @@ def save_zone_od_dist_matrix(self) -> None: print("Error: zone_od_dist_matrix does not exist. Please run calc_zone_od_distance_matrix() first.") return - path_output = gen_unique_filename(path2linux(os.path.join(self.output_dir, "zone_od_dist_matrix.csv"))) + path_output = generate_unique_filename(path2linux(os.path.join(self.output_dir, "zone_od_dist_matrix.csv"))) zone_od_dist_table_df = pd.DataFrame(self.zone_od_dist_matrix.values()) zone_od_dist_table_df = zone_od_dist_table_df[["o_zone_id", "o_zone_name", "d_zone_id", "d_zone_name", "dist_km", "geometry"]] diff --git a/grid2demand/func_lib/gen_agent_demand.py b/grid2demand/func_lib/gen_agent_demand.py index 2e39cce..e967dfd 100644 --- a/grid2demand/func_lib/gen_agent_demand.py +++ b/grid2demand/func_lib/gen_agent_demand.py @@ -6,9 +6,9 @@ ############################################################## import pandas as pd -from grid2demand.utils_lib.net_utils import Agent from random import choice, uniform import math +from pyufunc import gmns_geo def gen_agent_based_demand(node_dict: dict, zone_dict: dict, @@ -44,7 +44,7 @@ def gen_agent_based_demand(node_dict: dict, zone_dict: dict, departure_time = f"07{rand_time}" agent_lst.append( - Agent( + gmns_geo.Agent( id=i + 1, agent_type=agent_type, o_zone_id=o_zone_id, diff --git a/grid2demand/func_lib/gen_zone.py b/grid2demand/func_lib/gen_zone.py index 404aea0..01740a8 100644 --- a/grid2demand/func_lib/gen_zone.py +++ b/grid2demand/func_lib/gen_zone.py @@ -13,10 +13,10 @@ from multiprocessing import Pool, cpu_count from grid2demand.utils_lib.net_utils import Zone, Node -from grid2demand.utils_lib.utils import calc_distance_on_unit_sphere, int2alpha -from grid2demand.utils_lib.utils import func_running_time from grid2demand.utils_lib.pkg_settings import pkg_settings +from pyufunc import (calc_distance_on_unit_sphere, cvt_int_to_alpha, func_running_time) + @func_running_time def get_lng_lat_min_max(node_dict: dict[int, Node]) -> list: @@ -160,7 +160,7 @@ def generate_polygon(x_min, x_max, y_min, y_max) -> shapely.geometry.Polygon: y_max = y_block_maxmin_list[j][1] cell_polygon = generate_polygon(x_min, x_max, y_min, y_max) - row_alpha = int2alpha(j) + row_alpha = cvt_int_to_alpha(j) zone_dict[f"{row_alpha}{i}"] = Zone( id=zone_id_flag, name=f"{row_alpha}{i}", diff --git a/grid2demand/func_lib/read_node_poi.py b/grid2demand/func_lib/read_node_poi.py index 1e722b8..481c3c9 100644 --- a/grid2demand/func_lib/read_node_poi.py +++ b/grid2demand/func_lib/read_node_poi.py @@ -13,9 +13,9 @@ from grid2demand.utils_lib.net_utils import Node, POI, Zone from grid2demand.utils_lib.pkg_settings import pkg_settings -from grid2demand.utils_lib.utils import (func_running_time, path2linux, - get_filenames_from_folder_by_type, - check_required_files_exist) +from grid2demand.utils_lib.utils import check_required_files_exist +from pyufunc import (func_running_time, path2linux, + get_filenames_by_ext,) def create_node_from_dataframe(df_node: pd.DataFrame) -> dict[int, Node]: @@ -329,7 +329,7 @@ def read_network(input_folder: str = "", cpu_cores: int = 1) -> dict[str: dict]: raise FileNotFoundError(f"Input folder: {input_folder} does not exist.") # get all csv files in the folder - dir_files = get_filenames_from_folder_by_type(input_folder, "csv") + dir_files = get_filenames_by_ext(input_folder, "csv") # check if required files exist is_required_files_exist = check_required_files_exist(pkg_settings["required_files"], dir_files) diff --git a/grid2demand/func_lib/trip_rate_production_attraction.py b/grid2demand/func_lib/trip_rate_production_attraction.py index 4add7b6..43a7d77 100644 --- a/grid2demand/func_lib/trip_rate_production_attraction.py +++ b/grid2demand/func_lib/trip_rate_production_attraction.py @@ -8,8 +8,8 @@ import pandas as pd import os -from grid2demand.utils_lib.utils import path2linux from grid2demand.utils_lib.pkg_settings import pkg_settings +from pyufunc import path2linux def gen_poi_trip_rate(poi_dict: dict, diff --git a/requirements.txt b/requirements.txt index b606d4e..0ba013b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ numpy pandas shapely +pyufunc \ No newline at end of file From 8b4a0a2d7a5cb9928b4609904b967e20ecfd2edb Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:30:56 -0700 Subject: [PATCH 02/17] add tests --- doc/example_1/test_gd.py | 31 ------------------------------- tests/test_read_node.py | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 31 deletions(-) delete mode 100644 doc/example_1/test_gd.py create mode 100644 tests/test_read_node.py diff --git a/doc/example_1/test_gd.py b/doc/example_1/test_gd.py deleted file mode 100644 index ee0c326..0000000 --- a/doc/example_1/test_gd.py +++ /dev/null @@ -1,31 +0,0 @@ -import grid2demand_0205a as gd - - -"Step 1: Read Input Network Data" -net = gd.ReadNetworkFiles('') - -"Step 2: Partition Grid into cells" -zone = gd.PartitionGrid(number_of_x_blocks=5, number_of_y_blocks=5, cell_width=None, cell_height=None, - latitude=30) -# user can customize number of grid cells or cell's width and height - -"Step 3: Get Production/Attraction Rates of Each Land Use Type with a Specific Trip Purpose" -triprate = gd.GetPoiTripRate(trip_rate_folder='',trip_purpose=1) -# user can customize poi_trip_rate.csv and trip purpose - -"Step 4: Define Production/Attraction Value of Each Node According to POI Type" -nodedemand = gd.GetNodeDemand() - -"Step 5: Calculate Zone-to-zone Accessibility Matrix by Centroid-to-centroid Straight Distance" -accessibility = gd.ProduceAccessMatrix(latitude=30, accessibility_folder='') -# user can customize the latitude of the research area and accessibility.csv - -"Step 6: Apply Gravity Model to Conduct Trip Distribution" -demand = gd.RunGravityModel(trip_purpose=1, a=None, b=None, c=None) -# user can customize friction factor coefficients under a specific trip purpose -"Step 7: Generate Agent" -demand = gd.GenerateAgentBasedDemand() - - - - diff --git a/tests/test_read_node.py b/tests/test_read_node.py new file mode 100644 index 0000000..eefcf4d --- /dev/null +++ b/tests/test_read_node.py @@ -0,0 +1,16 @@ +# -*- coding:utf-8 -*- +############################################################## +# Created Date: Friday, March 1st 2024 +# Contact Info: luoxiangyong01@gmail.com +# Author/Copyright: Mr. Xiangyong Luo +############################################################## + + +import pytest +from grid2demand.func_lib.read_node_poi import read_node + + +def test_read_node_non_existing_file(): + # Test case for a non-existing node file + with pytest.raises(FileNotFoundError): + read_node(node_file="path/to/non_existing_node_file.csv") From b3b8e5be292afd0adbc87bbaf02b5b4b87948755 Mon Sep 17 00:00:00 2001 From: xyluo25 Date: Fri, 1 Mar 2024 16:32:07 -0700 Subject: [PATCH 03/17] Create python-package.yml --- .github/workflows/python-package.yml | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..28e07d1 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,40 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python package + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest pyufunc + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pytest From e71f19e63f1b4809714299c224b8c7d29b60f3da Mon Sep 17 00:00:00 2001 From: xyluo25 Date: Fri, 1 Mar 2024 16:34:51 -0700 Subject: [PATCH 04/17] Update python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 28e07d1..9b785f9 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -27,7 +27,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest pyufunc + python -m pip install flake8 pytest pyufunc grid2demand if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint with flake8 run: | From cb9017172cb5afad3c1c71454972c83efb322a6f Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:36:18 -0700 Subject: [PATCH 05/17] add requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0ba013b..a14e6c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ numpy pandas shapely -pyufunc \ No newline at end of file +pyufunc +urllib3 \ No newline at end of file From 61c8351bd4222a3a58f6c5cc11274f8d35dcfda2 Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:37:32 -0700 Subject: [PATCH 06/17] update requirements.txt --- requirements.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index a14e6c9..408d69c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,8 @@ -numpy -pandas -shapely -pyufunc +numpy>=1.26.4 +osm2gmns>=0.7.5 +pandas>=2.2.1 +pytest>=8.0.2 +pyufunc>=0.2.1 +setuptools>=68.2.2 +Shapely>=2.0.3 urllib3 \ No newline at end of file From c366113b4bf7d2da7936d4393b11c5213f918f01 Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:41:37 -0700 Subject: [PATCH 07/17] update requirements.txt --- .github/workflows/python-package.yml | 2 ++ requirements.txt | 3 --- requirements_dev.txt | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 requirements_dev.txt diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9b785f9..c00cc43 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -29,6 +29,8 @@ jobs: python -m pip install --upgrade pip python -m pip install flake8 pytest pyufunc grid2demand if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi + - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names diff --git a/requirements.txt b/requirements.txt index 408d69c..974b031 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,5 @@ numpy>=1.26.4 osm2gmns>=0.7.5 pandas>=2.2.1 -pytest>=8.0.2 pyufunc>=0.2.1 -setuptools>=68.2.2 Shapely>=2.0.3 -urllib3 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..1a3f0c0 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,21 @@ +/* + * Filename: c:\Users\roche\Anaconda_workspace\001_Github\grid2demand\requirements copy.txt + * Path: c:\Users\roche + * Created Date: Friday, March 1st 2024, 4:40:19 pm + * Author: Mr. Xiangyong Luo + * + * Copyright (c) 2024 Mr. Xiangyong Luo + */ + + +numpy>=1.26.4 +osm2gmns>=0.7.5 +pandas>=2.2.1 +pytest>=8.0.2 +pyufunc>=0.2.1 +setuptools>=68.2.2 +Shapely>=2.0.3 +urllib3 +requests +tqdm +beautifulsoup4 \ No newline at end of file From d68464bfed8abb6f7c4dce3d23b0c25eab4c9c0a Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:43:37 -0700 Subject: [PATCH 08/17] update --- requirements_dev.txt | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 1a3f0c0..651f9ed 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,13 +1,3 @@ -/* - * Filename: c:\Users\roche\Anaconda_workspace\001_Github\grid2demand\requirements copy.txt - * Path: c:\Users\roche - * Created Date: Friday, March 1st 2024, 4:40:19 pm - * Author: Mr. Xiangyong Luo - * - * Copyright (c) 2024 Mr. Xiangyong Luo - */ - - numpy>=1.26.4 osm2gmns>=0.7.5 pandas>=2.2.1 From 9edb6e86e49a4bc9161605a928c3545535cc19d7 Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:45:16 -0700 Subject: [PATCH 09/17] python 3.10+ --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c00cc43..ff2975e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 From 3debf293f7a426d4daf7940e20e810cdd31e8434 Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Tue, 26 Mar 2024 17:27:35 -0700 Subject: [PATCH 10/17] update input checking --- grid2demand/_grid2demand.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/grid2demand/_grid2demand.py b/grid2demand/_grid2demand.py index 0c8963f..4edfca9 100644 --- a/grid2demand/_grid2demand.py +++ b/grid2demand/_grid2demand.py @@ -77,7 +77,7 @@ def __check_input_dir(self) -> None: if is_optional_files_exist: print(f" : Optional files: {optional_files} are found in {self.input_dir}.") - print( " : Optional files could be used in the following steps.") + print(" : Optional files could be used in the following steps.") self.path_zone = path2linux(os.path.join(self.input_dir, "zone.csv")) print(" : Input directory is valid.\n") @@ -193,7 +193,6 @@ def taz2zone(self) -> dict[str, Zone]: \n Or you can use net2zone() to generate grid-based zones from node_dict.") return None - def sync_geometry_between_zone_and_node_poi(self, zone_dict: dict = "", node_dict: dict = "", poi_dict: dict = "") -> dict[str, dict]: """synchronize geometry between zone and node/poi. From 083b7cead39e15fa94826456680341221cc6e624 Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Tue, 26 Mar 2024 18:13:34 -0700 Subject: [PATCH 11/17] add one attribue in Node: __zone_id --- grid2demand/func_lib/read_node_poi.py | 10 +++++- grid2demand/utils_lib/net_utils.py | 48 +++++++++++++++------------ 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/grid2demand/func_lib/read_node_poi.py b/grid2demand/func_lib/read_node_poi.py index 481c3c9..d7ab06b 100644 --- a/grid2demand/func_lib/read_node_poi.py +++ b/grid2demand/func_lib/read_node_poi.py @@ -43,6 +43,13 @@ def create_node_from_dataframe(df_node: pd.DataFrame) -> dict[int, Node]: else: activity_location_tab = '' + # check zone_id field in node.csv + # if zone_id field exists and is not empty, assign it to __zone_id + try: + __zone_id = df_node.loc[i, 'zone_id'] + except Exception: + __zone_id = -1 + node = Node( id=df_node.loc[i, 'node_id'], activity_type=activity_type, @@ -51,7 +58,8 @@ def create_node_from_dataframe(df_node: pd.DataFrame) -> dict[int, Node]: y_coord=df_node.loc[i, 'y_coord'], poi_id=df_node.loc[i, 'poi_id'], boundary_flag=boundary_flag, - geometry=shapely.Point(df_node.loc[i, 'x_coord'], df_node.loc[i, 'y_coord']) + geometry=shapely.Point(df_node.loc[i, 'x_coord'], df_node.loc[i, 'y_coord']), + __zone_id=__zone_id ) node_dict[df_node.loc[i, 'node_id']] = node except Exception as e: diff --git a/grid2demand/utils_lib/net_utils.py b/grid2demand/utils_lib/net_utils.py index 97822d5..3467fe0 100644 --- a/grid2demand/utils_lib/net_utils.py +++ b/grid2demand/utils_lib/net_utils.py @@ -6,7 +6,7 @@ ############################################################## -from dataclasses import dataclass, field +from dataclasses import dataclass, field, asdict @dataclass @@ -26,6 +26,8 @@ class Node: activity_type: The activity type of the node. provided from osm2gmns such as motoway, residential, ... activity_location_tab: The activity location tab of the node. geometry: The geometry of the node. based on wkt format. + __zone_id: The zone ID. default == -1, + this will be assigned if field zone_id exists in the node.csv and is not empty """ id: int = 0 x_coord: float = 0 @@ -38,6 +40,10 @@ class Node: activity_type: str = '' activity_location_tab: str = '' geometry: str = '' + __zone_id: int = -1 + + def as_dict(self): + return asdict(self) @dataclass @@ -45,14 +51,14 @@ class POI: """A POI in the network. Attributes: - id: The POI ID. - x_coord: The x coordinate of the POI. - y_coord: The y coordinate of the POI. - count: The count of the POI. Total POI values for this POI node or POI zone - area: The area of the POI. Total area of polygon for this POI zone. unit is square meter + id : The POI ID. + x_coord : The x coordinate of the POI. + y_coord : The y coordinate of the POI. + count : The count of the POI. Total POI values for this POI node or POI zone + area : The area of the POI. Total area of polygon for this POI zone. unit is square meter poi_type: The type of the POI. Default is empty string geometry: The polygon of the POI. based on wkt format. Default is empty string - zone_id: The zone ID. mapping from zone + zone_id : The zone ID. mapping from zone """ id: int = 0 @@ -71,22 +77,22 @@ class Zone: """A zone in the network. Attributes: - id: The zone ID. - name: The name of the zone. - centroid_x: The centroid x coordinate of the zone. - centroid_y: The centroid y coordinate of the zone. - centroid: The centroid of the zone. (x, y) based on wkt format - x_max: The max x coordinate of the zone. - x_min: The min x coordinate of the zone. - y_max: The max y coordinate of the zone. - y_min: The min y coordinate of the zone. - node_id_list: Node IDs which belong to this zone. - poi_id_list: The POIs which belong to this zone. - production: The production of the zone. - attraction: The attraction of the zone. + id : The zone ID. + name : The name of the zone. + centroid_x : The centroid x coordinate of the zone. + centroid_y : The centroid y coordinate of the zone. + centroid : The centroid of the zone. (x, y) based on wkt format + x_max : The max x coordinate of the zone. + x_min : The min x coordinate of the zone. + y_max : The max y coordinate of the zone. + y_min : The min y coordinate of the zone. + node_id_list : Node IDs which belong to this zone. + poi_id_list : The POIs which belong to this zone. + production : The production of the zone. + attraction : The attraction of the zone. production_fixed: The fixed production of the zone (implement different models). attraction_fixed: The fixed attraction of the zone (implement different models). - geometry: The geometry of the zone. based on wkt format + geometry : The geometry of the zone. based on wkt format """ id: int = 0 From cb215b23bfb64fb9f1e5e4878518abca8eded795 Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Tue, 26 Mar 2024 18:49:02 -0700 Subject: [PATCH 12/17] add as_dict property for Node, POI, Link, Agent --- grid2demand/_grid2demand.py | 3 ++- grid2demand/func_lib/read_node_poi.py | 10 ++++++---- grid2demand/utils_lib/net_utils.py | 13 +++++++++++++ grid2demand/utils_lib/utils.py | 7 ++++--- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/grid2demand/_grid2demand.py b/grid2demand/_grid2demand.py index 4edfca9..60ae4b2 100644 --- a/grid2demand/_grid2demand.py +++ b/grid2demand/_grid2demand.py @@ -63,6 +63,7 @@ def __check_input_dir(self) -> None: # check required files in input directory dir_files = get_filenames_by_ext(self.input_dir, "csv") required_files = pkg_settings.get("required_files", []) + is_required_files_exist = check_required_files_exist(required_files, dir_files) if not is_required_files_exist: raise Exception(f"Error: Required files are not satisfied. Please check {required_files} in {self.input_dir}.") @@ -73,7 +74,7 @@ def __check_input_dir(self) -> None: # check optional files in input directory (zone.csv) optional_files = pkg_settings.get("optional_files", []) - is_optional_files_exist = check_required_files_exist(optional_files, dir_files) + is_optional_files_exist = check_required_files_exist(optional_files, dir_files, verbose=False) if is_optional_files_exist: print(f" : Optional files: {optional_files} are found in {self.input_dir}.") diff --git a/grid2demand/func_lib/read_node_poi.py b/grid2demand/func_lib/read_node_poi.py index d7ab06b..5feeeed 100644 --- a/grid2demand/func_lib/read_node_poi.py +++ b/grid2demand/func_lib/read_node_poi.py @@ -43,10 +43,10 @@ def create_node_from_dataframe(df_node: pd.DataFrame) -> dict[int, Node]: else: activity_location_tab = '' - # check zone_id field in node.csv + # check whether zone_id field in node.csv or not # if zone_id field exists and is not empty, assign it to __zone_id try: - __zone_id = df_node.loc[i, 'zone_id'] + __zone_id = df_node.loc[i, 'zone_id'] or -1 except Exception: __zone_id = -1 @@ -97,7 +97,6 @@ def read_node(node_file: str = "", cpu_cores: int = 1) -> dict[int: Node]: if not os.path.exists(node_file): raise FileNotFoundError(f"File: {node_file} does not exist.") - print(f" : Parallel creating Nodes using Pool with {cpu_cores} CPUs. Please wait...") # read node.csv with specified columns and chunksize for iterations node_required_cols = pkg_settings["node_required_fields"] chunk_size = pkg_settings["data_chunk_size"] @@ -105,13 +104,16 @@ def read_node(node_file: str = "", cpu_cores: int = 1) -> dict[int: Node]: \n and chunksize {chunk_size} for iterations...") df_node_chunk = pd.read_csv(node_file, usecols=node_required_cols, chunksize=chunk_size) - # Parallel processing using Pool + print(f" : Parallel creating Nodes using Pool with {cpu_cores} CPUs. Please wait...") node_dict_final = {} + + # Parallel processing using Pool with Pool(cpu_cores) as pool: results = pool.map(create_node_from_dataframe, df_node_chunk) for node_dict in results: node_dict_final.update(node_dict) + print(f" : Successfully loaded node.csv: {len(node_dict_final)} Nodes loaded.") return node_dict_final diff --git a/grid2demand/utils_lib/net_utils.py b/grid2demand/utils_lib/net_utils.py index 3467fe0..6d0f5d0 100644 --- a/grid2demand/utils_lib/net_utils.py +++ b/grid2demand/utils_lib/net_utils.py @@ -42,6 +42,7 @@ class Node: geometry: str = '' __zone_id: int = -1 + @property def as_dict(self): return asdict(self) @@ -71,6 +72,10 @@ class POI: geometry: str = '' zone_id: int = 0 + @property + def as_dict(self): + return asdict(self) + @dataclass class Zone: @@ -112,6 +117,10 @@ class Zone: attraction_fixed: float = 0 geometry: str = '' + @property + def as_dict(self): + return asdict(self) + @dataclass class Agent: @@ -156,3 +165,7 @@ class Agent: b_complete_trip: bool = False geometry: str = '' departure_time: int = 0 # unit is second + + @property + def as_dict(self): + return asdict(self) diff --git a/grid2demand/utils_lib/utils.py b/grid2demand/utils_lib/utils.py index 46b5a5a..96b3909 100644 --- a/grid2demand/utils_lib/utils.py +++ b/grid2demand/utils_lib/utils.py @@ -201,7 +201,7 @@ def get_filenames_from_folder_by_type(dir_name: str, file_type: str = "txt", isT os.path.join(dir_name, file)) for file in os.listdir(dir_name) if file.split(".")[-1] == file_type] -def check_required_files_exist(required_files: list, dir_files: list) -> bool: +def check_required_files_exist(required_files: list, dir_files: list, verbose: bool = True) -> bool: # format the required file name to standard linux path required_files = [path2linux(os.path.abspath(filename)) for filename in required_files] @@ -215,8 +215,9 @@ def check_required_files_exist(required_files: list, dir_files: list) -> bool: if all(mask): return True - print(f" : Error: Required files are not satisfied, \ - missing files are: {[required_files_short[i] for i in range(len(required_files_short)) if not mask[i]]}") + if verbose: + print(f" : Error: Required files are not satisfied, \ + missing files are: {[required_files_short[i] for i in range(len(required_files_short)) if not mask[i]]}") return False From 78f686351d9f3603fd22232ee21576a09ffd4ef1 Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Wed, 27 Mar 2024 01:15:03 -0700 Subject: [PATCH 13/17] edit net2zone to enable generate zone by zone_id from node.csv --- grid2demand/func_lib/gen_zone.py | 19 +++++++++++++++++-- grid2demand/func_lib/read_node_poi.py | 18 +++++++++++++++--- grid2demand/utils_lib/net_utils.py | 4 ++-- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/grid2demand/func_lib/gen_zone.py b/grid2demand/func_lib/gen_zone.py index 01740a8..e6da40a 100644 --- a/grid2demand/func_lib/gen_zone.py +++ b/grid2demand/func_lib/gen_zone.py @@ -6,6 +6,7 @@ ############################################################## from __future__ import absolute_import +import contextlib import itertools import pandas as pd import shapely @@ -49,7 +50,7 @@ def net2zone(node_dict: dict[int, Node], num_x_blocks: int = 0, num_y_blocks: int = 0, cell_width: float = 0, - cell_height: float = 0, unit: str = "km") -> dict[str, Zone]: + cell_height: float = 0, unit: str = "km", use_zone_id: bool = False) -> dict[str, Zone]: """convert node_dict to zone_dict by grid. The grid can be defined by num_x_blocks and num_y_blocks, or cell_width and cell_height. if num_x_blocks and num_y_blocks are specified, the grid will be divided into num_x_blocks * num_y_blocks. @@ -63,7 +64,7 @@ def net2zone(node_dict: dict[int, Node], num_y_blocks (int, optional): total number of blocks/grids from y direction. Defaults to 10. cell_width (float, optional): the width for each block/grid . Defaults to 0. unit: km. cell_height (float, optional): the height for each block/grid. Defaults to 0. unit: km. - unit (str, optional): the unit of cell_width and cell_height. Defaults to "km". + unit (str, optional): the unit of cell_width and cell_height. Defaults to "km". Options:"meter", "km", "mile". Raises ValueError: Please provide num_x_blocks and num_y_blocks or cell_width and cell_height @@ -87,6 +88,20 @@ def net2zone(node_dict: dict[int, Node], # coord_y_min, coord_y_max = df_node['y_coord'].min( # ) - 0.000001, df_node['y_coord'].max() + 0.000001 + # generate zone based on zone_id in node.csv + if use_zone_id: + node_dict_zone_id = {} + + for node_id in node_dict: + with contextlib.suppress(AttributeError): + if node_dict[node_id]._zone_id != -1: + node_dict_zone_id[node_id] = node_dict[node_id] + + if not node_dict_zone_id: + print(" : No zone_id found in node_dict, will generate zone based on original node_dict") + else: + node_dict = node_dict_zone_id + coord_x_min, coord_x_max, coord_y_min, coord_y_max = get_lng_lat_min_max(node_dict) # Priority: num_x_blocks, number_y_blocks > cell_width, cell_height diff --git a/grid2demand/func_lib/read_node_poi.py b/grid2demand/func_lib/read_node_poi.py index 5feeeed..3fd9546 100644 --- a/grid2demand/func_lib/read_node_poi.py +++ b/grid2demand/func_lib/read_node_poi.py @@ -46,9 +46,14 @@ def create_node_from_dataframe(df_node: pd.DataFrame) -> dict[int, Node]: # check whether zone_id field in node.csv or not # if zone_id field exists and is not empty, assign it to __zone_id try: - __zone_id = df_node.loc[i, 'zone_id'] or -1 + _zone_id = df_node.loc[i, 'zone_id'] + + # check if _zone is none or empty, assign -1 + if pd.isna(_zone_id) or not _zone_id: + _zone_id = -1 + except Exception: - __zone_id = -1 + _zone_id = -1 node = Node( id=df_node.loc[i, 'node_id'], @@ -59,7 +64,7 @@ def create_node_from_dataframe(df_node: pd.DataFrame) -> dict[int, Node]: poi_id=df_node.loc[i, 'poi_id'], boundary_flag=boundary_flag, geometry=shapely.Point(df_node.loc[i, 'x_coord'], df_node.loc[i, 'y_coord']), - __zone_id=__zone_id + _zone_id=_zone_id ) node_dict[df_node.loc[i, 'node_id']] = node except Exception as e: @@ -100,6 +105,13 @@ def read_node(node_file: str = "", cpu_cores: int = 1) -> dict[int: Node]: # read node.csv with specified columns and chunksize for iterations node_required_cols = pkg_settings["node_required_fields"] chunk_size = pkg_settings["data_chunk_size"] + + # read first two rows to check whether required fields are in node.csv + df_node_2rows = pd.read_csv(node_file, nrows=2) + col_names = df_node_2rows.columns.tolist() + if "zone_id" in col_names: + node_required_cols.append("zone_id") + print(f" : Reading node.csv with specified columns: {node_required_cols} \ \n and chunksize {chunk_size} for iterations...") df_node_chunk = pd.read_csv(node_file, usecols=node_required_cols, chunksize=chunk_size) diff --git a/grid2demand/utils_lib/net_utils.py b/grid2demand/utils_lib/net_utils.py index 6d0f5d0..8437934 100644 --- a/grid2demand/utils_lib/net_utils.py +++ b/grid2demand/utils_lib/net_utils.py @@ -26,7 +26,7 @@ class Node: activity_type: The activity type of the node. provided from osm2gmns such as motoway, residential, ... activity_location_tab: The activity location tab of the node. geometry: The geometry of the node. based on wkt format. - __zone_id: The zone ID. default == -1, + _zone_id: The zone ID. default == -1, this will be assigned if field zone_id exists in the node.csv and is not empty """ id: int = 0 @@ -40,7 +40,7 @@ class Node: activity_type: str = '' activity_location_tab: str = '' geometry: str = '' - __zone_id: int = -1 + _zone_id: int = -1 @property def as_dict(self): From 6d2a73b2a0c4b60476a1799d206b7b815b2d4583 Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Wed, 27 Mar 2024 01:56:20 -0700 Subject: [PATCH 14/17] add node and zone pair --- grid2demand/_grid2demand.py | 16 ++++++++++++++-- grid2demand/func_lib/gen_agent_demand.py | 2 +- grid2demand/func_lib/gravity_model.py | 5 +++-- grid2demand_tutorial.py | 2 +- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/grid2demand/_grid2demand.py b/grid2demand/_grid2demand.py index 60ae4b2..4b482f1 100644 --- a/grid2demand/_grid2demand.py +++ b/grid2demand/_grid2demand.py @@ -112,6 +112,15 @@ def load_node(self) -> dict[int, Node]: raise FileNotFoundError(f"Error: File {self.path_node} does not exist.") self.node_dict = read_node(self.path_node, self.pkg_settings.get("set_cpu_cores")) + + # generate node_zone_pair {node_id: zone_id} for later use + node_zone_pair = { + node_id: self.node_dict[node_id]._zone_id + for node_id in self.node_dict + if self.node_dict[node_id]._zone_id != -1 + } + self.node_zone_pair = node_zone_pair + return self.node_dict @property @@ -150,7 +159,7 @@ def load_network(self) -> dict[str, dict]: return network_dict def net2zone(self, node_dict: dict[int, Node], num_x_blocks: int = 10, num_y_blocks: int = 10, - cell_width: float = 0, cell_height: float = 0, unit: str = "km") -> dict[str, Zone]: + cell_width: float = 0, cell_height: float = 0, unit: str = "km", use_zone_id: bool = False) -> dict[str, Zone]: """convert node_dict to zone_dict by grid. The grid can be defined by num_x_blocks and num_y_blocks, or cell_width and cell_height. if num_x_blocks and num_y_blocks are specified, the grid will be divided into num_x_blocks * num_y_blocks. @@ -165,6 +174,8 @@ def net2zone(self, node_dict: dict[int, Node], num_x_blocks: int = 10, num_y_blo cell_width (float, optional): the width for each block/grid . Defaults to 0. unit: km. cell_height (float, optional): the height for each block/grid. Defaults to 0. unit: km. unit (str, optional): the unit of cell_width and cell_height. Defaults to "km". + Options: ["km", "meter", "mile"] + use_zone_id (bool, optional): whether to use zone_id from node_dict. Defaults to False. Returns: dict[str, Zone]: zone_dict {zone_name: Zone} @@ -350,7 +361,8 @@ def run_gravity_model(self, zone_dict: dict = "", zone_dict = self.zone_dict zone_od_dist_matrix = self.zone_od_dist_matrix - self.df_demand = run_gravity_model(zone_dict, zone_od_dist_matrix) + self.zone_od_demand_matrix = run_gravity_model(zone_dict, zone_od_dist_matrix, trip_purpose, alpha, beta, gamma) + self.df_demand = pd.DataFrame(list(self.zone_od_demand_matrix.values())) return self.df_demand def gen_agent_based_demand(self, node_dict: dict = "", zone_dict: dict = "", diff --git a/grid2demand/func_lib/gen_agent_demand.py b/grid2demand/func_lib/gen_agent_demand.py index e967dfd..175c021 100644 --- a/grid2demand/func_lib/gen_agent_demand.py +++ b/grid2demand/func_lib/gen_agent_demand.py @@ -58,4 +58,4 @@ def gen_agent_based_demand(node_dict: dict, zone_dict: dict, ) ) print(" : Successfully generated agent-based demand data.") - return pd.DataFrame(agent_lst) \ No newline at end of file + return pd.DataFrame(agent_lst) diff --git a/grid2demand/func_lib/gravity_model.py b/grid2demand/func_lib/gravity_model.py index 3a63180..9699325 100644 --- a/grid2demand/func_lib/gravity_model.py +++ b/grid2demand/func_lib/gravity_model.py @@ -37,7 +37,7 @@ def run_gravity_model(zone_dict: dict, trip_purpose: int = 1, alpha: float = 28507, beta: float = -0.02, - gamma: float = -0.123) -> pd.DataFrame: + gamma: float = -0.123) -> dict: # if trip purpose is specified in trip_purpose_dict, use the default value # otherwise, use the user-specified value trip_purpose_dict = pkg_settings.get("trip_purpose_dict") @@ -68,4 +68,5 @@ def run_gravity_model(zone_dict: dict, # Generate demand.csv print(" : Successfully run gravity model to generate demand.csv.") - return pd.DataFrame(list(zone_od_matrix_dict.values())) \ No newline at end of file + # return pd.DataFrame(list(zone_od_matrix_dict.values())) + return zone_od_matrix_dict diff --git a/grid2demand_tutorial.py b/grid2demand_tutorial.py index d7185cb..49a4659 100644 --- a/grid2demand_tutorial.py +++ b/grid2demand_tutorial.py @@ -21,7 +21,7 @@ # Step 2: Generate zone dictionary from node dictionary # by specifying number of x blocks and y blocks - zone_dict = gd.net2zone(node_dict, num_x_blocks=10, num_y_blocks=10) + zone_dict = gd.net2zone(node_dict, num_x_blocks=10, num_y_blocks=10, use_zone_id=True) # Step 2: Generate zone based on grid size with 10 km width and 10km height for each zone # zone_dict = gd.net2zone(node_dict, cell_width=10, cell_height=10) From 75a1c30160bde8136538e5644c59d7f3d78d9896 Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Wed, 27 Mar 2024 03:54:24 -0700 Subject: [PATCH 15/17] add zone_id based demand --- grid2demand/_grid2demand.py | 60 ++++++++++++++++++++++++--- grid2demand/func_lib/gravity_model.py | 8 ++-- grid2demand_tutorial.py | 4 +- 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/grid2demand/_grid2demand.py b/grid2demand/_grid2demand.py index 4b482f1..3bf6bea 100644 --- a/grid2demand/_grid2demand.py +++ b/grid2demand/_grid2demand.py @@ -6,7 +6,9 @@ ############################################################## import os +from itertools import combinations import pandas as pd +import shapely from grid2demand.utils_lib.pkg_settings import pkg_settings from grid2demand.utils_lib.net_utils import Node, POI, Zone, Agent from grid2demand.utils_lib.utils import check_required_files_exist @@ -21,7 +23,7 @@ from grid2demand.func_lib.gravity_model import run_gravity_model, calc_zone_production_attraction from grid2demand.func_lib.gen_agent_demand import gen_agent_based_demand -from pyufunc import (path2linux, get_filenames_by_ext, +from pyufunc import (path2linux, get_filenames_by_ext, cvt_int_to_alpha, generate_unique_filename) @@ -119,7 +121,7 @@ def load_node(self) -> dict[int, Node]: for node_id in self.node_dict if self.node_dict[node_id]._zone_id != -1 } - self.node_zone_pair = node_zone_pair + self.node_zone_id_pair = node_zone_pair return self.node_dict @@ -156,10 +158,19 @@ def load_network(self) -> dict[str, dict]: network_dict = read_network(self.input_dir, self.pkg_settings.get("set_cpu_cores")) self.node_dict = network_dict.get('node_dict') self.poi_dict = network_dict.get('poi_dict') + + # generate node_zone_pair {node_id: zone_id} for later use + node_zone_pair = { + node_id: self.node_dict[node_id]._zone_id + for node_id in self.node_dict + if self.node_dict[node_id]._zone_id != -1 + } + self.node_zone_id_pair = node_zone_pair return network_dict def net2zone(self, node_dict: dict[int, Node], num_x_blocks: int = 10, num_y_blocks: int = 10, - cell_width: float = 0, cell_height: float = 0, unit: str = "km", use_zone_id: bool = False) -> dict[str, Zone]: + cell_width: float = 0, cell_height: float = 0, + unit: str = "km", use_zone_id: bool = False) -> dict[str, Zone]: """convert node_dict to zone_dict by grid. The grid can be defined by num_x_blocks and num_y_blocks, or cell_width and cell_height. if num_x_blocks and num_y_blocks are specified, the grid will be divided into num_x_blocks * num_y_blocks. @@ -186,6 +197,7 @@ def net2zone(self, node_dict: dict[int, Node], num_x_blocks: int = 10, num_y_blo print(" : Generating zone dictionary...") self.zone_dict = net2zone(node_dict, num_x_blocks, num_y_blocks, cell_width, cell_height, unit) + self.zone_id_name_pair = {Zone.id: Zone.name for Zone in self.zone_dict.values()} return self.zone_dict def taz2zone(self) -> dict[str, Zone]: @@ -197,6 +209,7 @@ def taz2zone(self) -> dict[str, Zone]: if self.path_zone: print(" : Generating zone dictionary...") self.zone_dict = read_zone(self.path_zone, self.pkg_settings.get("set_cpu_cores")) + self.zone_id_name_pair = {Zone.id: Zone.name for Zone in self.zone_dict.values()} return self.zone_dict @@ -343,7 +356,8 @@ def run_gravity_model(self, zone_dict: dict = "", trip_purpose: int = 1, alpha: float = 28507, beta: float = -0.02, - gamma: float = -0.123) -> pd.DataFrame: + gamma: float = -0.123, + use_zone_id: bool = False) -> pd.DataFrame: """run gravity model to generate demand Args: @@ -361,8 +375,42 @@ def run_gravity_model(self, zone_dict: dict = "", zone_dict = self.zone_dict zone_od_dist_matrix = self.zone_od_dist_matrix - self.zone_od_demand_matrix = run_gravity_model(zone_dict, zone_od_dist_matrix, trip_purpose, alpha, beta, gamma) - self.df_demand = pd.DataFrame(list(self.zone_od_demand_matrix.values())) + self.zone_od_dist_matrix = run_gravity_model(zone_dict, zone_od_dist_matrix, trip_purpose, alpha, beta, gamma) + self.df_demand = pd.DataFrame(list(self.zone_od_dist_matrix.values())) + + if use_zone_id: + comb = combinations(self.node_zone_id_pair.keys(), 2) + comb_grid_zone_name = {} + + for pair in comb: + o_node_id, d_node_id = pair + o_zone_id = self.node_dict[o_node_id].zone_id + d_zone_id = self.node_dict[d_node_id].zone_id + o_zone_name = self.zone_id_name_pair[o_zone_id] + d_zone_name = self.zone_id_name_pair[d_zone_id] + if o_zone_name != d_zone_name: + try: + comb_grid_zone_name[(o_zone_name, d_zone_name)] = self.zone_od_dist_matrix[( + o_zone_name, d_zone_name)] + except KeyError: + print(f" : Error: zone_od_dist_matrix does not contain {o_zone_name, d_zone_name}.") + + demand_lst = [] + for zone_od_pair in comb_grid_zone_name: + demand_lst.append( + { + "o_node_id": o_node_id, + "o_zone_id": self.node_zone_id_pair[o_node_id], + "d_node_id": d_node_id, + "d_zone_id": self.node_zone_id_pair[d_node_id], + "volume": self.zone_od_dist_matrix[zone_od_pair]['volume'], + "geometry": shapely.LineString([self.node_dict[o_node_id].geometry, + self.node_dict[d_node_id].geometry]) + + } + ) + return pd.DataFrame(demand_lst) + return self.df_demand def gen_agent_based_demand(self, node_dict: dict = "", zone_dict: dict = "", diff --git a/grid2demand/func_lib/gravity_model.py b/grid2demand/func_lib/gravity_model.py index 9699325..d3cc825 100644 --- a/grid2demand/func_lib/gravity_model.py +++ b/grid2demand/func_lib/gravity_model.py @@ -33,7 +33,7 @@ def calc_zone_od_friction_attraction(zone_od_friction_matrix_dict: dict, zone_di def run_gravity_model(zone_dict: dict, - zone_od_matrix_dict: dict, + zone_od_dist_matrix: dict, trip_purpose: int = 1, alpha: float = 28507, beta: float = -0.02, @@ -53,7 +53,7 @@ def run_gravity_model(zone_dict: dict, # perform zone od friction matrix zone_od_friction_matrix_dict = { zone_name_pair: alpha * (od_dist["dist_km"] ** beta) * (np.exp(od_dist["dist_km"] * gamma)) if od_dist["dist_km"] != 0 else 0 - for zone_name_pair, od_dist in zone_od_matrix_dict.items() + for zone_name_pair, od_dist in zone_od_dist_matrix.items() } # perform attraction calculation @@ -61,7 +61,7 @@ def run_gravity_model(zone_dict: dict, # perform od trip flow (volume) calculation for zone_name_pair in zone_od_friction_matrix_dict: - zone_od_matrix_dict[zone_name_pair]["volume"] = float(zone_dict[zone_name_pair[0]].production * + zone_od_dist_matrix[zone_name_pair]["volume"] = float(zone_dict[zone_name_pair[0]].production * zone_dict[zone_name_pair[1]].attraction * zone_od_friction_matrix_dict[zone_name_pair] / zone_od_friction_attraction_dict[zone_name_pair[0]]) @@ -69,4 +69,4 @@ def run_gravity_model(zone_dict: dict, # Generate demand.csv print(" : Successfully run gravity model to generate demand.csv.") # return pd.DataFrame(list(zone_od_matrix_dict.values())) - return zone_od_matrix_dict + return zone_od_dist_matrix diff --git a/grid2demand_tutorial.py b/grid2demand_tutorial.py index 49a4659..9ee9df6 100644 --- a/grid2demand_tutorial.py +++ b/grid2demand_tutorial.py @@ -40,7 +40,7 @@ zone_dict_update, node_dict_update, poi_dict_update = updated_dict.values() # Step 4: Calculate zone-to-zone od distance matrix - zone_od_distance_matrix = gd.calc_zone_od_distance_matrix(zone_dict_update) + zone_od_dist_matrix = gd.calc_zone_od_distance_matrix(zone_dict_update) # Step 5: Generate poi trip rate for each poi poi_trip_rate = gd.gen_poi_trip_rate(poi_dict_update) @@ -52,7 +52,7 @@ zone_prod_attr = gd.calc_zone_prod_attr(node_prod_attr, zone_dict_update) # Step 7: Run gravity model to generate agent-based demand - df_demand = gd.run_gravity_model(zone_prod_attr, zone_od_distance_matrix) + df_demand = gd.run_gravity_model(zone_prod_attr, zone_od_dist_matrix, use_zone_id=True) # Step 8: generate agent-based demand df_agent = gd.gen_agent_based_demand(node_prod_attr, zone_prod_attr, df_demand=df_demand) From 409cc28eee15ca7937659a73d2eb41dabba74715 Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Wed, 27 Mar 2024 04:01:23 -0700 Subject: [PATCH 16/17] update tutorials --- grid2demand_tutorial.py | 13 +---- grid2demand_tutorial_use_zone_in_nodeCSV.py | 61 +++++++++++++++++++++ grid2demand_tutorial_user_TAZ.py | 61 +++++++++++++++++++++ 3 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 grid2demand_tutorial_use_zone_in_nodeCSV.py create mode 100644 grid2demand_tutorial_user_TAZ.py diff --git a/grid2demand_tutorial.py b/grid2demand_tutorial.py index 9ee9df6..11624c7 100644 --- a/grid2demand_tutorial.py +++ b/grid2demand_tutorial.py @@ -21,18 +21,11 @@ # Step 2: Generate zone dictionary from node dictionary # by specifying number of x blocks and y blocks - zone_dict = gd.net2zone(node_dict, num_x_blocks=10, num_y_blocks=10, use_zone_id=True) + zone_dict = gd.net2zone(node_dict, num_x_blocks=10, num_y_blocks=10) - # Step 2: Generate zone based on grid size with 10 km width and 10km height for each zone + # Step 2: Generate zone based on grid size with 10 km width and 10 km height for each zone # zone_dict = gd.net2zone(node_dict, cell_width=10, cell_height=10) - # Step 2: Generate zone dictionary from zone.csv file - # please note, if you use this function, - # you need to make sure the zone.csv file is in the input directory - # this will generate zones based on your own TAZs - # and zone.csv requires at least two columns: zone_id and geometry - # zone_dict = gd.taz2zone() - # Step 3: synchronize geometry info between zone, node and poi # add zone_id to node and poi dictionaries # also add node_list and poi_list to zone dictionary @@ -52,7 +45,7 @@ zone_prod_attr = gd.calc_zone_prod_attr(node_prod_attr, zone_dict_update) # Step 7: Run gravity model to generate agent-based demand - df_demand = gd.run_gravity_model(zone_prod_attr, zone_od_dist_matrix, use_zone_id=True) + df_demand = gd.run_gravity_model(zone_prod_attr, zone_od_dist_matrix) # Step 8: generate agent-based demand df_agent = gd.gen_agent_based_demand(node_prod_attr, zone_prod_attr, df_demand=df_demand) diff --git a/grid2demand_tutorial_use_zone_in_nodeCSV.py b/grid2demand_tutorial_use_zone_in_nodeCSV.py new file mode 100644 index 0000000..ff6b96e --- /dev/null +++ b/grid2demand_tutorial_use_zone_in_nodeCSV.py @@ -0,0 +1,61 @@ +# -*- coding:utf-8 -*- +############################################################## +# Created Date: Monday, September 11th 2023 +# Contact Info: luoxiangyong01@gmail.com +# Author/Copyright: Mr. Xiangyong Luo +############################################################## + +from __future__ import absolute_import +from grid2demand import GRID2DEMAND + +if __name__ == "__main__": + + # Step 0: Specify input directory, if not, use current working directory as default input directory + input_dir = "./datasets/dubai" + + # Initialize a GRID2DEMAND object + gd = GRID2DEMAND(input_dir) + + # Step 1: Load node and poi data from input directory + node_dict, poi_dict = gd.load_network.values() + + # Step 2: Generate zone dictionary from node dictionary + # by specifying number of x blocks and y blocks + zone_dict = gd.net2zone(node_dict, num_x_blocks=10, num_y_blocks=10, use_zone_id=True) + + # Step 2: Generate zone based on grid size with 10 km width and 10km height for each zone + # zone_dict = gd.net2zone(node_dict, cell_width=10, cell_height=10, use_zone_id=True) + + # Step 3: synchronize geometry info between zone, node and poi + # add zone_id to node and poi dictionaries + # also add node_list and poi_list to zone dictionary + updated_dict = gd.sync_geometry_between_zone_and_node_poi(zone_dict, node_dict, poi_dict) + zone_dict_update, node_dict_update, poi_dict_update = updated_dict.values() + + # Step 4: Calculate zone-to-zone od distance matrix + zone_od_dist_matrix = gd.calc_zone_od_distance_matrix(zone_dict_update) + + # Step 5: Generate poi trip rate for each poi + poi_trip_rate = gd.gen_poi_trip_rate(poi_dict_update) + + # Step 6: Generate node production attraction for each node based on poi_trip_rate + node_prod_attr = gd.gen_node_prod_attr(node_dict_update, poi_trip_rate) + + # Step 6.1: Calculate zone production and attraction based on node production and attraction + zone_prod_attr = gd.calc_zone_prod_attr(node_prod_attr, zone_dict_update) + + # Step 7: Run gravity model to generate agent-based demand + df_demand = gd.run_gravity_model(zone_prod_attr, zone_od_dist_matrix, use_zone_id=True) + + # Step 8: generate agent-based demand + df_agent = gd.gen_agent_based_demand(node_prod_attr, zone_prod_attr, df_demand=df_demand) + + # You can also view and edit the package setting by using gd.pkg_settings + print(gd.pkg_settings) + + # Step 9: Output demand, agent, zone, zone_od_dist_table, zone_od_dist_matrix files + gd.save_demand + gd.save_agent + gd.save_zone + gd.save_zone_od_dist_table + gd.save_zone_od_dist_matrix \ No newline at end of file diff --git a/grid2demand_tutorial_user_TAZ.py b/grid2demand_tutorial_user_TAZ.py new file mode 100644 index 0000000..192dd6f --- /dev/null +++ b/grid2demand_tutorial_user_TAZ.py @@ -0,0 +1,61 @@ +# -*- coding:utf-8 -*- +############################################################## +# Created Date: Monday, September 11th 2023 +# Contact Info: luoxiangyong01@gmail.com +# Author/Copyright: Mr. Xiangyong Luo +############################################################## + +from __future__ import absolute_import +from grid2demand import GRID2DEMAND + +if __name__ == "__main__": + + # Step 0: Specify input directory, if not, use current working directory as default input directory + input_dir = "./datasets/dubai" + + # Initialize a GRID2DEMAND object + gd = GRID2DEMAND(input_dir) + + # Step 1: Load node and poi data from input directory + node_dict, poi_dict = gd.load_network.values() + + # Step 2: Generate zone dictionary from zone.csv file + # please note, if you use this function, + # you need to make sure the zone.csv file is in the input directory + # this will generate zones based on your own TAZs + # and zone.csv requires at least two columns: zone_id and geometry + zone_dict = gd.taz2zone() + + # Step 3: synchronize geometry info between zone, node and poi + # add zone_id to node and poi dictionaries + # also add node_list and poi_list to zone dictionary + updated_dict = gd.sync_geometry_between_zone_and_node_poi(zone_dict, node_dict, poi_dict) + zone_dict_update, node_dict_update, poi_dict_update = updated_dict.values() + + # Step 4: Calculate zone-to-zone od distance matrix + zone_od_dist_matrix = gd.calc_zone_od_distance_matrix(zone_dict_update) + + # Step 5: Generate poi trip rate for each poi + poi_trip_rate = gd.gen_poi_trip_rate(poi_dict_update) + + # Step 6: Generate node production attraction for each node based on poi_trip_rate + node_prod_attr = gd.gen_node_prod_attr(node_dict_update, poi_trip_rate) + + # Step 6.1: Calculate zone production and attraction based on node production and attraction + zone_prod_attr = gd.calc_zone_prod_attr(node_prod_attr, zone_dict_update) + + # Step 7: Run gravity model to generate agent-based demand + df_demand = gd.run_gravity_model(zone_prod_attr, zone_od_dist_matrix) + + # Step 8: generate agent-based demand + df_agent = gd.gen_agent_based_demand(node_prod_attr, zone_prod_attr, df_demand=df_demand) + + # You can also view and edit the package setting by using gd.pkg_settings + print(gd.pkg_settings) + + # Step 9: Output demand, agent, zone, zone_od_dist_table, zone_od_dist_matrix files + gd.save_demand + gd.save_agent + gd.save_zone + gd.save_zone_od_dist_table + gd.save_zone_od_dist_matrix \ No newline at end of file From 14b8ec7195995e40a78fb0e0fb54b1711551ae09 Mon Sep 17 00:00:00 2001 From: xyluo <36498464+Xiangyongluo@users.noreply.github.com> Date: Sat, 30 Mar 2024 21:02:25 -0700 Subject: [PATCH 17/17] v0.4.1 --- grid2demand/__init__.py | 2 +- grid2demand/_grid2demand.py | 221 ++++++++++++------ grid2demand/func_lib/gen_agent_demand.py | 11 +- grid2demand/func_lib/gen_zone.py | 55 +++-- grid2demand/func_lib/gravity_model.py | 23 +- grid2demand/func_lib/read_node_poi.py | 66 ++++-- .../trip_rate_production_attraction.py | 17 +- grid2demand_tutorial_use_zone_in_nodeCSV.py | 13 +- 8 files changed, 284 insertions(+), 124 deletions(-) diff --git a/grid2demand/__init__.py b/grid2demand/__init__.py index 32779e3..cc9f953 100644 --- a/grid2demand/__init__.py +++ b/grid2demand/__init__.py @@ -7,7 +7,7 @@ from .utils_lib.pkg_settings import pkg_settings from ._grid2demand import GRID2DEMAND -print('grid2demand, version 0.4.0') +print('grid2demand, version 0.4.1') __all__ = ["read_node", "read_poi", "read_network", diff --git a/grid2demand/_grid2demand.py b/grid2demand/_grid2demand.py index 3bf6bea..bd9697c 100644 --- a/grid2demand/_grid2demand.py +++ b/grid2demand/_grid2demand.py @@ -29,7 +29,16 @@ class GRID2DEMAND: - def __init__(self, input_dir: str = "", output_dir: str = "") -> None: + def __init__(self, input_dir: str = "", output_dir: str = "", + use_zone_id: bool = False, verbose: bool = False) -> None: + """initialize GRID2DEMAND object + + Args: + input_dir (str, optional): input directory of your data. Defaults to "". + output_dir (str, optional): output directory. Defaults to "". + use_zone_id (bool, optional): whether to use zone_id from node.csv. Defaults to False. + verbose (bool, optional): print processing message. Defaults to False. + """ # check input directory if not input_dir: @@ -41,6 +50,9 @@ def __init__(self, input_dir: str = "", output_dir: str = "") -> None: self.input_dir = path2linux(input_dir) self.output_dir = path2linux(output_dir) if output_dir else self.input_dir + self.verbose = verbose + self.use_zone_id = use_zone_id + # check input directory self.__check_input_dir() @@ -58,7 +70,8 @@ def __check_input_dir(self) -> None: None: will generate self.path_node and self.path_poi for class instance. """ - print(" : Checking input directory...") + if self.verbose: + print(" : Checking input directory...") if not os.path.isdir(self.input_dir): raise NotADirectoryError(f"Error: Input directory {self.input_dir} does not exist.") @@ -66,7 +79,7 @@ def __check_input_dir(self) -> None: dir_files = get_filenames_by_ext(self.input_dir, "csv") required_files = pkg_settings.get("required_files", []) - is_required_files_exist = check_required_files_exist(required_files, dir_files) + is_required_files_exist = check_required_files_exist(required_files, dir_files, verbose=self.verbose) if not is_required_files_exist: raise Exception(f"Error: Required files are not satisfied. Please check {required_files} in {self.input_dir}.") @@ -83,12 +96,16 @@ def __check_input_dir(self) -> None: print(" : Optional files could be used in the following steps.") self.path_zone = path2linux(os.path.join(self.input_dir, "zone.csv")) - print(" : Input directory is valid.\n") + if self.verbose: + print(" : Input directory is valid.\n") def __load_pkg_settings(self) -> None: - print(" : Loading default package settings...") + if self.verbose: + print(" : Loading default package settings...") self.pkg_settings = pkg_settings - print(" : Package settings loaded successfully.\n") + + if self.verbose: + print(" : Package settings loaded successfully.\n") @property def load_node(self) -> dict[int, Node]: @@ -113,15 +130,19 @@ def load_node(self) -> dict[int, Node]: if not os.path.exists(self.path_node): raise FileNotFoundError(f"Error: File {self.path_node} does not exist.") - self.node_dict = read_node(self.path_node, self.pkg_settings.get("set_cpu_cores")) + # add zone_id to node_dict if use_zone_id is True + if self.use_zone_id and "zone_id" not in pkg_settings["node_required_fields"]: + pkg_settings["node_required_fields"].append("zone_id") + + self.node_dict = read_node(self.path_node, self.pkg_settings.get("set_cpu_cores"), verbose=self.verbose) # generate node_zone_pair {node_id: zone_id} for later use - node_zone_pair = { + # the zone_id based on node.csv in field zone_id + self.__pair_node_zone_id = { node_id: self.node_dict[node_id]._zone_id for node_id in self.node_dict if self.node_dict[node_id]._zone_id != -1 } - self.node_zone_id_pair = node_zone_pair return self.node_dict @@ -139,7 +160,7 @@ def load_poi(self) -> dict[int, POI]: if not os.path.exists(self.path_poi): raise FileExistsError(f"Error: File {self.path_poi} does not exist.") - self.poi_dict = read_poi(self.path_poi, self.pkg_settings.get("set_cpu_cores")) + self.poi_dict = read_poi(self.path_poi, self.pkg_settings.get("set_cpu_cores"), verbose=self.verbose) return self.poi_dict @property @@ -155,22 +176,28 @@ def load_network(self) -> dict[str, dict]: if not os.path.isdir(self.input_dir): raise FileExistsError(f"Error: Input directory {self.input_dir} does not exist.") - network_dict = read_network(self.input_dir, self.pkg_settings.get("set_cpu_cores")) + + # add zone_id to node_dict if use_zone_id is True + if self.use_zone_id and "zone_id" not in pkg_settings["node_required_fields"]: + pkg_settings["node_required_fields"].append("zone_id") + + network_dict = read_network(self.input_dir, self.pkg_settings.get("set_cpu_cores"), verbose=self.verbose) self.node_dict = network_dict.get('node_dict') self.poi_dict = network_dict.get('poi_dict') # generate node_zone_pair {node_id: zone_id} for later use - node_zone_pair = { + # the zone_id based on node.csv in field zone_id + self.__pair_node_zone_id = { node_id: self.node_dict[node_id]._zone_id for node_id in self.node_dict if self.node_dict[node_id]._zone_id != -1 } - self.node_zone_id_pair = node_zone_pair + return network_dict def net2zone(self, node_dict: dict[int, Node], num_x_blocks: int = 10, num_y_blocks: int = 10, cell_width: float = 0, cell_height: float = 0, - unit: str = "km", use_zone_id: bool = False) -> dict[str, Zone]: + unit: str = "km") -> dict[str, Zone]: """convert node_dict to zone_dict by grid. The grid can be defined by num_x_blocks and num_y_blocks, or cell_width and cell_height. if num_x_blocks and num_y_blocks are specified, the grid will be divided into num_x_blocks * num_y_blocks. @@ -191,25 +218,36 @@ def net2zone(self, node_dict: dict[int, Node], num_x_blocks: int = 10, num_y_blo Returns: dict[str, Zone]: zone_dict {zone_name: Zone} """ - print(" : Note: This method will generate grid-based zones from node_dict. \ - \n : If you want to use your own zones(TAZs), \ - \n : please skip this method and use taz2zone() instead. \n") + if self.verbose: + print(" : Note: net2zone will generate grid-based zones from node_dict. \ + \n : If you want to use your own zones(TAZs), \ + \n : please skip this method and use taz2zone() instead. \n") print(" : Generating zone dictionary...") - self.zone_dict = net2zone(node_dict, num_x_blocks, num_y_blocks, cell_width, cell_height, unit) - self.zone_id_name_pair = {Zone.id: Zone.name for Zone in self.zone_dict.values()} + self.zone_dict, self.node_dict_within_zone = net2zone(node_dict, + num_x_blocks, + num_y_blocks, + cell_width, + cell_height, + unit, + self.use_zone_id, + verbose=self.verbose) + self.__pair_zone_id_name = {Zone.id: Zone.name for Zone in self.zone_dict.values()} return self.zone_dict def taz2zone(self) -> dict[str, Zone]: - print(" : Note: This method will generate zones from zone.csv (TAZs). \ - \n : If you want to use grid-based zones (generate zones from node_dict) , \ - \n : please skip this method and use net2zone() instead. \n") + if self.verbose: + print(" : Note: taz2zone will generate zones from zone.csv (TAZs). \ + \n : If you want to use grid-based zones (generate zones from node_dict) , \ + \n : please skip this method and use net2zone() instead. \n") if self.path_zone: - print(" : Generating zone dictionary...") + if self.verbose: + print(" : Generating zone dictionary...") self.zone_dict = read_zone(self.path_zone, self.pkg_settings.get("set_cpu_cores")) - self.zone_id_name_pair = {Zone.id: Zone.name for Zone in self.zone_dict.values()} + self.__pair_zone_id_name = { + Zone.id: Zone.name for Zone in self.zone_dict.values()} return self.zone_dict @@ -237,7 +275,8 @@ def sync_geometry_between_zone_and_node_poi(self, zone_dict: dict = "", Returns: dict[str, dict]: {"zone_dict": self.zone_dict, "node_dict": self.node_dict, "poi_dict": self.poi_dict} """ - print(" : Synchronizing geometry between zone and node/poi...") + if self.verbose: + print(" : Synchronizing geometry between zone and node/poi...") # if not specified, use self.zone_dict, self.node_dict, self.poi_dict as input if not all([zone_dict, node_dict, poi_dict]): @@ -247,7 +286,7 @@ def sync_geometry_between_zone_and_node_poi(self, zone_dict: dict = "", # synchronize zone with node try: - zone_node_dict = sync_zone_and_node_geometry(zone_dict, node_dict, self.pkg_settings.get("set_cpu_cores")) + zone_node_dict = sync_zone_and_node_geometry(zone_dict, node_dict, self.pkg_settings.get("set_cpu_cores"), verbose=self.verbose) zone_dict_add_node = zone_node_dict.get('zone_dict') self.node_dict = zone_node_dict.get('node_dict') except Exception as e: @@ -286,7 +325,9 @@ def calc_zone_od_distance_matrix(self, zone_dict: dict = "") -> dict[tuple, floa if not zone_dict: zone_dict = self.zone_dict - self.zone_od_dist_matrix = calc_zone_od_matrix(zone_dict, self.pkg_settings.get("set_cpu_cores")) + self.zone_od_dist_matrix = calc_zone_od_matrix(zone_dict, + self.pkg_settings.get("set_cpu_cores"), + verbose=self.verbose) return self.zone_od_dist_matrix def gen_poi_trip_rate(self, poi_dict: dict = "", trip_rate_file: str = "", trip_purpose: int = 1) -> dict[int, POI]: @@ -313,7 +354,7 @@ def gen_poi_trip_rate(self, poi_dict: dict = "", trip_rate_file: str = "", trip_ raise Exception(f" : Error: trip_rate_file {trip_rate_file} must be a csv file.") self.pkg_settings["trip_rate_file"] = pd.read_csv(trip_rate_file) - self.poi_dict = gen_poi_trip_rate(poi_dict, trip_rate_file, trip_purpose) + self.poi_dict = gen_poi_trip_rate(poi_dict, trip_rate_file, trip_purpose, verbose=self.verbose) return self.poi_dict def gen_node_prod_attr(self, node_dict: dict = "", poi_dict: dict = "") -> dict[int, Node]: @@ -331,7 +372,7 @@ def gen_node_prod_attr(self, node_dict: dict = "", poi_dict: dict = "") -> dict[ node_dict = self.node_dict poi_dict = self.poi_dict - self.node_dict = gen_node_prod_attr(node_dict, poi_dict) + self.node_dict = gen_node_prod_attr(node_dict, poi_dict,verbose=self.verbose) return self.node_dict def calc_zone_prod_attr(self, node_dict: dict = "", zone_dict: dict = "") -> dict[str, Zone]: @@ -347,7 +388,7 @@ def calc_zone_prod_attr(self, node_dict: dict = "", zone_dict: dict = "") -> dic if not all([node_dict, zone_dict]): node_dict = self.node_dict zone_dict = self.zone_dict - self.zone = calc_zone_production_attraction(node_dict, zone_dict) + self.zone = calc_zone_production_attraction(node_dict, zone_dict, verbose=self.verbose) return self.zone @@ -356,60 +397,87 @@ def run_gravity_model(self, zone_dict: dict = "", trip_purpose: int = 1, alpha: float = 28507, beta: float = -0.02, - gamma: float = -0.123, - use_zone_id: bool = False) -> pd.DataFrame: + gamma: float = -0.123) -> pd.DataFrame: """run gravity model to generate demand Args: - zone_dict (dict, optional): _description_. Defaults to "". - zone_od_dist_matrix (dict, optional): _description_. Defaults to "". - trip_purpose (int, optional): _description_. Defaults to 1. - alpha (float, optional): _description_. Defaults to 28507. - beta (float, optional): _description_. Defaults to -0.02. - gamma (float, optional): _description_. Defaults to -0.123. + zone_dict (dict, optional): dict store zones info. Defaults to "". + zone_od_dist_matrix (dict, optional): OD distance matrix. Defaults to "". + trip_purpose (int, optional): purpose of trip. Defaults to 1. + alpha (float, optional): parameter alpha. Defaults to 28507. + beta (float, optional): parameter beta. Defaults to -0.02. + gamma (float, optional): parameter gamma. Defaults to -0.123. Returns: pd.DataFrame: the final demand dataframe """ + # if not specified, use self.zone_dict, self.zone_od_dist_matrix as input if not all([zone_dict, zone_od_dist_matrix]): zone_dict = self.zone_dict zone_od_dist_matrix = self.zone_od_dist_matrix - self.zone_od_dist_matrix = run_gravity_model(zone_dict, zone_od_dist_matrix, trip_purpose, alpha, beta, gamma) + # run gravity model to generate demand + self.zone_od_dist_matrix = run_gravity_model(zone_dict, zone_od_dist_matrix, trip_purpose, alpha, beta, gamma, verbose=self.verbose) self.df_demand = pd.DataFrame(list(self.zone_od_dist_matrix.values())) - if use_zone_id: - comb = combinations(self.node_zone_id_pair.keys(), 2) - comb_grid_zone_name = {} + # if use_zone_id is True, generate demand dataframe based on zone_id from node.csv + if self.use_zone_id: + # generate comb {(o_node_id, d_node_id)} + # pair_node_zone_id: node_id and zone_id from node.csv, they are unique pairs + # comb: all possible combinations of node_id pairs + # comb: can be seen as all possible zone_id pairs from node.csv + node_zone_lst = list(self.__pair_node_zone_id.keys()) + + # create all possible combinations of node_id pairs + comb = [(i, j) for i in node_zone_lst for j in node_zone_lst] + + # generate demand_lst to store demand dataframe + demand_lst = [] + + # create demand dictionary for pair in comb: o_node_id, d_node_id = pair - o_zone_id = self.node_dict[o_node_id].zone_id - d_zone_id = self.node_dict[d_node_id].zone_id - o_zone_name = self.zone_id_name_pair[o_zone_id] - d_zone_name = self.zone_id_name_pair[d_zone_id] + o_zone_id = self.node_dict[o_node_id].zone_id # zone_id from generated zone + d_zone_id = self.node_dict[d_node_id].zone_id # zone_id from generated zone + + o_zone_name = self.__pair_zone_id_name[o_zone_id] # zone_name from generated zone + d_zone_name = self.__pair_zone_id_name[d_zone_id] # zone_name from generated zone if o_zone_name != d_zone_name: try: - comb_grid_zone_name[(o_zone_name, d_zone_name)] = self.zone_od_dist_matrix[( - o_zone_name, d_zone_name)] - except KeyError: - print(f" : Error: zone_od_dist_matrix does not contain {o_zone_name, d_zone_name}.") + dist_km = self.zone_od_dist_matrix[(o_zone_name, d_zone_name)].get('dist_km') + volume = self.zone_od_dist_matrix[(o_zone_name, d_zone_name)].get('volume') + + demand_lst.append( + { + # "o_node_id": o_node_id, + "o_zone_id": self.__pair_node_zone_id[o_node_id], + "o_zone_name": self.__pair_node_zone_id[o_node_id], + + # "d_node_id": d_node_id, + "d_zone_id": self.__pair_node_zone_id[d_node_id], + "d_zone_name": self.__pair_node_zone_id[d_node_id], + "dist_km": dist_km, + "volume": volume, + "geometry": shapely.LineString([self.node_dict[o_node_id].geometry, + self.node_dict[d_node_id].geometry]) + + } + ) - demand_lst = [] - for zone_od_pair in comb_grid_zone_name: - demand_lst.append( - { - "o_node_id": o_node_id, - "o_zone_id": self.node_zone_id_pair[o_node_id], - "d_node_id": d_node_id, - "d_zone_id": self.node_zone_id_pair[d_node_id], - "volume": self.zone_od_dist_matrix[zone_od_pair]['volume'], - "geometry": shapely.LineString([self.node_dict[o_node_id].geometry, - self.node_dict[d_node_id].geometry]) - - } - ) - return pd.DataFrame(demand_lst) + except KeyError: + dist_km = 0 + volume = 0 + + if self.verbose: + print(f" : Error: zone_od_dist_matrix does not contain {o_zone_name, d_zone_name}.") + if demand_lst: + self.df_demand_based_zone_id = pd.DataFrame(demand_lst) + else: + self.df_demand_based_zone_id = pd.DataFrame(columns=["o_zone_id", "o_zone_name", + "d_zone_id", "d_zone_name", + "dist_km", "volume", "geometry"]) + return self.df_demand_based_zone_id return self.df_demand @@ -433,7 +501,7 @@ def gen_agent_based_demand(self, node_dict: dict = "", zone_dict: dict = "", node_dict = self.node_dict zone_dict = self.zone_dict - self.df_agent = gen_agent_based_demand(node_dict, zone_dict, df_demand=df_demand) + self.df_agent = gen_agent_based_demand(node_dict, zone_dict, df_demand=df_demand, verbose=self.verbose) return self.df_agent @property @@ -444,9 +512,17 @@ def save_demand(self) -> None: return path_output = generate_unique_filename(path2linux(os.path.join(self.output_dir, "demand.csv"))) + + if self.use_zone_id: + df_demand_non_zero = self.df_demand_based_zone_id[self.df_demand_based_zone_id["volume"] > 0] + df_demand_non_zero.to_csv(path_output, index=False) + print(f" : Successfully saved demand.csv to {self.output_dir}") + return None + df_demand_non_zero = self.df_demand[self.df_demand["volume"] > 0] df_demand_non_zero.to_csv(path_output, index=False) print(f" : Successfully saved demand.csv to {self.output_dir}") + return None @property def save_agent(self) -> None: @@ -499,4 +575,15 @@ def save_zone_od_dist_matrix(self) -> None: zone_od_dist_matrix_df = zone_od_dist_table_df.pivot(index='o_zone_name', columns='d_zone_name', values='dist_km') zone_od_dist_matrix_df.to_csv(path_output) - print(f" : Successfully saved zone_od_dist_matrix.csv to {self.output_dir}") \ No newline at end of file + print(f" : Successfully saved zone_od_dist_matrix.csv to {self.output_dir}") + + @property + def save_node_within_zone(self): + if not hasattr(self, "node_dict_within_zone"): + print("Error: node_dict_within_zone does not exist. Please run net2zone() first.") + return + + path_output = generate_unique_filename(path2linux(os.path.join(self.output_dir, "node_within_zone.csv"))) + node_within_zone_df = pd.DataFrame(self.node_dict_within_zone.values()) + node_within_zone_df.to_csv(path_output, index=False) + print(f" : Successfully saved node_within_zone.csv to {self.output_dir}") \ No newline at end of file diff --git a/grid2demand/func_lib/gen_agent_demand.py b/grid2demand/func_lib/gen_agent_demand.py index 175c021..49bda83 100644 --- a/grid2demand/func_lib/gen_agent_demand.py +++ b/grid2demand/func_lib/gen_agent_demand.py @@ -12,8 +12,10 @@ def gen_agent_based_demand(node_dict: dict, zone_dict: dict, - path_demand: str = "", df_demand: pd.DataFrame = "", - agent_type: str = "v") -> pd.DataFrame: + path_demand: str = "", + df_demand: pd.DataFrame = "", + agent_type: str = "v", + verbose: bool = False) -> pd.DataFrame: # either path_demand or df_demand must be provided # if path_demand is provided, read demand data from path_demand @@ -57,5 +59,8 @@ def gen_agent_based_demand(node_dict: dict, zone_dict: dict, departure_time=departure_time ) ) - print(" : Successfully generated agent-based demand data.") + + if verbose: + print(" :Successfully generated agent-based demand data.") + return pd.DataFrame(agent_lst) diff --git a/grid2demand/func_lib/gen_zone.py b/grid2demand/func_lib/gen_zone.py index e6da40a..5c7aeb8 100644 --- a/grid2demand/func_lib/gen_zone.py +++ b/grid2demand/func_lib/gen_zone.py @@ -29,8 +29,10 @@ def get_lng_lat_min_max(node_dict: dict[int, Node]) -> list: Returns: list: [min_lng, max_lng, min_lat, max_lat] """ - coord_x_min, coord_x_max = node_dict[0].x_coord, node_dict[0].x_coord - coord_y_min, coord_y_max = node_dict[0].y_coord, node_dict[0].y_coord + first_key = list(node_dict.keys())[0] + + coord_x_min, coord_x_max = node_dict[first_key].x_coord, node_dict[first_key].x_coord + coord_y_min, coord_y_max = node_dict[first_key].y_coord, node_dict[first_key].y_coord for node_id in node_dict: if node_dict[node_id].x_coord < coord_x_min: @@ -50,7 +52,9 @@ def net2zone(node_dict: dict[int, Node], num_x_blocks: int = 0, num_y_blocks: int = 0, cell_width: float = 0, - cell_height: float = 0, unit: str = "km", use_zone_id: bool = False) -> dict[str, Zone]: + cell_height: float = 0, unit: str = "km", + use_zone_id: bool = False, + verbose: bool = False) -> dict[str, Zone]: """convert node_dict to zone_dict by grid. The grid can be defined by num_x_blocks and num_y_blocks, or cell_width and cell_height. if num_x_blocks and num_y_blocks are specified, the grid will be divided into num_x_blocks * num_y_blocks. @@ -65,6 +69,8 @@ def net2zone(node_dict: dict[int, Node], cell_width (float, optional): the width for each block/grid . Defaults to 0. unit: km. cell_height (float, optional): the height for each block/grid. Defaults to 0. unit: km. unit (str, optional): the unit of cell_width and cell_height. Defaults to "km". Options:"meter", "km", "mile". + use_zone_id (bool, optional): whether to use zone_id in node_dict. Defaults to False. + verbose (bool, optional): print processing information. Defaults to False. Raises ValueError: Please provide num_x_blocks and num_y_blocks or cell_width and cell_height @@ -104,6 +110,16 @@ def net2zone(node_dict: dict[int, Node], coord_x_min, coord_x_max, coord_y_min, coord_y_max = get_lng_lat_min_max(node_dict) + # get nodes within the boundary + if use_zone_id: + node_dict_within_boundary = {} + for node_id in node_dict: + if node_dict[node_id].x_coord >= coord_x_min and node_dict[node_id].x_coord <= coord_x_max and \ + node_dict[node_id].y_coord >= coord_y_min and node_dict[node_id].y_coord <= coord_y_max: + node_dict_within_boundary[node_id] = node_dict[node_id] + else: + node_dict_within_boundary = node_dict + # Priority: num_x_blocks, number_y_blocks > cell_width, cell_height # if num_x_blocks and num_y_blocks are given, use them to partition the study area # else if cell_width and cell_height are given, use them to partition the study area @@ -227,9 +243,11 @@ def generate_polygon(x_min, x_max, y_min, y_max) -> shapely.geometry.Polygon: geometry=points_lst[i] ) zone_id_flag += 1 - print(f" : Successfully generated zone dictionary: {len(zone_dict) - 4 * len(zone_upper_row)} Zones generated, \ - \n plus {4 * len(zone_upper_row)} boundary gates (points))") - return zone_dict + + if verbose: + print(f" : Successfully generated zone dictionary: {len(zone_dict) - 4 * len(zone_upper_row)} Zones generated, \ + \n plus {4 * len(zone_upper_row)} boundary gates (points))") + return (zone_dict, node_dict_within_boundary) def sync_node_with_zones(args: tuple) -> tuple: @@ -250,7 +268,7 @@ def sync_node_with_zones(args: tuple) -> tuple: @func_running_time -def sync_zone_and_node_geometry(zone_dict: dict, node_dict: dict, cpu_cores: int = 1) -> dict: +def sync_zone_and_node_geometry(zone_dict: dict, node_dict: dict, cpu_cores: int = 1, verbose: bool = False) -> dict: """Map nodes to zone cells Parameters @@ -262,7 +280,8 @@ def sync_zone_and_node_geometry(zone_dict: dict, node_dict: dict, cpu_cores: int """ # Create a pool of worker processes - print(f" : Parallel synchronizing Nodes and Zones using Pool with {cpu_cores} CPUs. Please wait...") + if verbose: + print(f" : Parallel synchronizing Nodes and Zones using Pool with {cpu_cores} CPUs. Please wait...") # Prepare arguments for the pool args_list = [(node_id, node, zone_dict) for node_id, node in node_dict.items()] @@ -276,7 +295,9 @@ def sync_zone_and_node_geometry(zone_dict: dict, node_dict: dict, cpu_cores: int zone_dict[zone_name].node_id_list.append(node_id) node_dict[node_id] = node - print(" : Successfully synchronized zone and node geometry") + if verbose: + print(" : Successfully synchronized zone and node geometry") + return {"node_dict": node_dict, "zone_dict": zone_dict} @@ -298,7 +319,7 @@ def sync_poi_with_zones(args: tuple) -> tuple: @func_running_time -def sync_zone_and_poi_geometry(zone_dict: dict, poi_dict: dict, cpu_cores: int = 1) -> dict: +def sync_zone_and_poi_geometry(zone_dict: dict, poi_dict: dict, cpu_cores: int = 1, verbose: bool = False) -> dict: """Synchronize zone cells and POIs to update zone_id attribute for POIs and poi_id_list attribute for zone cells Args: @@ -310,7 +331,8 @@ def sync_zone_and_poi_geometry(zone_dict: dict, poi_dict: dict, cpu_cores: int = """ # Create a pool of worker processes - print(f" : Parallel synchronizing POIs and Zones using Pool with {cpu_cores} CPUs. Please wait...") + if verbose: + print(f" : Parallel synchronizing POIs and Zones using Pool with {cpu_cores} CPUs. Please wait...") # Prepare arguments for the pool args_list = [(poi_id, poi, zone_dict) for poi_id, poi in poi_dict.items()] @@ -324,7 +346,8 @@ def sync_zone_and_poi_geometry(zone_dict: dict, poi_dict: dict, cpu_cores: int = zone_dict[zone_name].poi_id_list.append(poi_id) poi_dict[poi_id] = poi - print(" : Successfully synchronized zone and poi geometry") + if verbose: + print(" : Successfully synchronized zone and poi geometry") return {"poi_dict": poi_dict, "zone_dict": zone_dict} @@ -349,7 +372,7 @@ def distance_calculation(args: tuple) -> tuple: @func_running_time -def calc_zone_od_matrix(zone_dict: dict, cpu_cores: int = 1) -> dict[tuple[str, str], dict]: +def calc_zone_od_matrix(zone_dict: dict, cpu_cores: int = 1, verbose: bool = False) -> dict[tuple[str, str], dict]: """Calculate the zone-to-zone distance matrix Args: @@ -369,7 +392,8 @@ def calc_zone_od_matrix(zone_dict: dict, cpu_cores: int = 1) -> dict[tuple[str, len_df_zone = len(df_zone) # Prepare arguments for the pool - print(f" : Parallel calculating zone-to-zone distance matrix using Pool with {cpu_cores} CPUs. Please wait...") + if verbose: + print(f" : Parallel calculating zone-to-zone distance matrix using Pool with {cpu_cores} CPUs. Please wait...") args_list = [(i, j, df_zone) for i, j in itertools.product(range(len_df_zone), range(len_df_zone))] with Pool(processes=cpu_cores) as pool: @@ -379,5 +403,6 @@ def calc_zone_od_matrix(zone_dict: dict, cpu_cores: int = 1) -> dict[tuple[str, # Gather results dist_dict = dict(results) - print(" : Successfully calculated zone-to-zone distance matrix") + if verbose: + print(" : Successfully calculated zone-to-zone distance matrix") return dist_dict diff --git a/grid2demand/func_lib/gravity_model.py b/grid2demand/func_lib/gravity_model.py index d3cc825..a219d38 100644 --- a/grid2demand/func_lib/gravity_model.py +++ b/grid2demand/func_lib/gravity_model.py @@ -10,25 +10,31 @@ import pandas as pd -def calc_zone_production_attraction(node_dict: dict, zone_dict: dict) -> dict: +def calc_zone_production_attraction(node_dict: dict, zone_dict: dict, verbose: bool = False) -> dict: # calculate zone production and attraction based on node production and attraction for zone_name in zone_dict: if zone_dict[zone_name].node_id_list: for node_id in zone_dict[zone_name].node_id_list: zone_dict[zone_name].production += node_dict[node_id].production zone_dict[zone_name].attraction += node_dict[node_id].attraction - print(" : Successfully calculated zone production and attraction based on node production and attraction.") + + if verbose: + print(" : Successfully calculated zone production and attraction based on node production and attraction.") + return zone_dict -def calc_zone_od_friction_attraction(zone_od_friction_matrix_dict: dict, zone_dict: dict) -> dict: +def calc_zone_od_friction_attraction(zone_od_friction_matrix_dict: dict, zone_dict: dict, verbose: bool = False) -> dict: zone_od_friction_attraction_dict = {} for zone_name, friction_val in zone_od_friction_matrix_dict.items(): if zone_name[0] not in zone_od_friction_attraction_dict: zone_od_friction_attraction_dict[zone_name[0]] = friction_val * zone_dict[zone_name[1]].attraction else: zone_od_friction_attraction_dict[zone_name[0]] += friction_val * zone_dict[zone_name[1]].attraction - print(" : Successfully calculated zone od friction attraction.") + + if verbose: + print(" : Successfully calculated zone od friction attraction.") + return zone_od_friction_attraction_dict @@ -37,7 +43,8 @@ def run_gravity_model(zone_dict: dict, trip_purpose: int = 1, alpha: float = 28507, beta: float = -0.02, - gamma: float = -0.123) -> dict: + gamma: float = -0.123, + verbose: bool = False) -> dict: # if trip purpose is specified in trip_purpose_dict, use the default value # otherwise, use the user-specified value trip_purpose_dict = pkg_settings.get("trip_purpose_dict") @@ -57,7 +64,7 @@ def run_gravity_model(zone_dict: dict, } # perform attraction calculation - zone_od_friction_attraction_dict = calc_zone_od_friction_attraction(zone_od_friction_matrix_dict, zone_dict) + zone_od_friction_attraction_dict = calc_zone_od_friction_attraction(zone_od_friction_matrix_dict, zone_dict, verbose=verbose) # perform od trip flow (volume) calculation for zone_name_pair in zone_od_friction_matrix_dict: @@ -67,6 +74,8 @@ def run_gravity_model(zone_dict: dict, zone_od_friction_attraction_dict[zone_name_pair[0]]) # Generate demand.csv - print(" : Successfully run gravity model to generate demand.csv.") + if verbose: + print(" : Successfully run gravity model to generate demand.csv.") + # return pd.DataFrame(list(zone_od_matrix_dict.values())) return zone_od_dist_matrix diff --git a/grid2demand/func_lib/read_node_poi.py b/grid2demand/func_lib/read_node_poi.py index 3fd9546..0b6dde4 100644 --- a/grid2demand/func_lib/read_node_poi.py +++ b/grid2demand/func_lib/read_node_poi.py @@ -73,11 +73,13 @@ def create_node_from_dataframe(df_node: pd.DataFrame) -> dict[int, Node]: @func_running_time -def read_node(node_file: str = "", cpu_cores: int = 1) -> dict[int: Node]: +def read_node(node_file: str = "", cpu_cores: int = 1, verbose: bool = False) -> dict[int: Node]: """Read node.csv file and return a dict of nodes. Args: node_file (str, optional): node file path. Defaults to "". + cpu_cores (int, optional): number of cpu cores for parallel processing. Defaults to 1. + verbose (bool, optional): print processing information. Defaults to False. Raises: FileNotFoundError: File: {node_file} does not exist. @@ -112,11 +114,13 @@ def read_node(node_file: str = "", cpu_cores: int = 1) -> dict[int: Node]: if "zone_id" in col_names: node_required_cols.append("zone_id") - print(f" : Reading node.csv with specified columns: {node_required_cols} \ - \n and chunksize {chunk_size} for iterations...") + if verbose: + print(f" : Reading node.csv with specified columns: {node_required_cols} \ + \n and chunksize {chunk_size} for iterations...") df_node_chunk = pd.read_csv(node_file, usecols=node_required_cols, chunksize=chunk_size) - print(f" : Parallel creating Nodes using Pool with {cpu_cores} CPUs. Please wait...") + if verbose: + print(f" : Parallel creating Nodes using Pool with {cpu_cores} CPUs. Please wait...") node_dict_final = {} # Parallel processing using Pool @@ -126,7 +130,8 @@ def read_node(node_file: str = "", cpu_cores: int = 1) -> dict[int: Node]: for node_dict in results: node_dict_final.update(node_dict) - print(f" : Successfully loaded node.csv: {len(node_dict_final)} Nodes loaded.") + if verbose: + print(f" : Successfully loaded node.csv: {len(node_dict_final)} Nodes loaded.") return node_dict_final @@ -164,11 +169,13 @@ def create_poi_from_dataframe(df_poi: pd.DataFrame) -> dict[int, POI]: @func_running_time -def read_poi(poi_file: str = "", cpu_cores: int = 1) -> dict[int: POI]: +def read_poi(poi_file: str = "", cpu_cores: int = 1, verbose: bool = False) -> dict[int: POI]: """Read poi.csv file and return a dict of POIs. Args: poi_file (str): The poi.csv file path. default is "". + cpu_cores (int, optional): number of cpu cores for parallel processing. Defaults to 1. + verbose (bool, optional): print processing information. Defaults to False. Raises: FileNotFoundError: if poi_file does not exist. @@ -197,12 +204,15 @@ def read_poi(poi_file: str = "", cpu_cores: int = 1) -> dict[int: POI]: # Read poi.csv with specified columns and chunksize for iterations poi_required_cols = pkg_settings["poi_required_fields"] chunk_size = pkg_settings["data_chunk_size"] - print(f" : Reading poi.csv with specified columns: {poi_required_cols} \ - \n and chunksize {chunk_size} for iterations...") + + if verbose: + print(f" : Reading poi.csv with specified columns: {poi_required_cols} \ + \n and chunksize {chunk_size} for iterations...") df_poi_chunk = pd.read_csv(poi_file, usecols=poi_required_cols, chunksize=chunk_size) # Parallel processing using Pool - print(f" : Parallel creating POIs using Pool with {cpu_cores} CPUs. Please wait...") + if verbose: + print(f" : Parallel creating POIs using Pool with {cpu_cores} CPUs. Please wait...") poi_dict_final = {} with Pool(cpu_cores) as pool: @@ -211,7 +221,9 @@ def read_poi(poi_file: str = "", cpu_cores: int = 1) -> dict[int: POI]: for poi_dict in results: poi_dict_final.update(poi_dict) - print(f" : Successfully loaded poi.csv: {len(poi_dict_final)} POIs loaded.") + if verbose: + print(f" : Successfully loaded poi.csv: {len(poi_dict_final)} POIs loaded.") + return poi_dict_final @@ -262,13 +274,18 @@ def create_zone_from_dataframe(df_zone: pd.DataFrame) -> dict[int, Zone]: @func_running_time -def read_zone(zone_file: str = "", cpu_cores: int = 1) -> dict[int: Zone]: - """_summary_ +def read_zone(zone_file: str = "", cpu_cores: int = 1, verbose: bool = False) -> dict[int: Zone]: + """Read zone.csv file and return a dict of Zones. Raises: FileNotFoundError: _description_ FileNotFoundError: _description_ + Args: + zone_file (str, optional): the input zone file path. Defaults to "". + cpu_cores (int, optional): number of cpu cores for parallel processing. Defaults to 1. + verbose (bool, optional): print processing information. Defaults to False. + Returns: _type_: _description_ """ @@ -283,8 +300,10 @@ def read_zone(zone_file: str = "", cpu_cores: int = 1) -> dict[int: Zone]: # load default settings for zone required fields and chunk size zone_required_cols = pkg_settings["zone_required_fields"] chunk_size = pkg_settings["data_chunk_size"] - print(f" : Reading zone.csv with specified columns: {zone_required_cols} \ - \n and chunksize {chunk_size} for iterations...") + + if verbose: + print(f" : Reading zone.csv with specified columns: {zone_required_cols} \ + \n and chunksize {chunk_size} for iterations...") # check whether required fields are in zone.csv df_zone = pd.read_csv(zone_file, nrows=1) @@ -298,7 +317,8 @@ def read_zone(zone_file: str = "", cpu_cores: int = 1) -> dict[int: Zone]: df_zone_chunk = pd.read_csv(zone_file, usecols=zone_required_cols, chunksize=chunk_size) # Parallel processing using Pool - print(f" : Parallel creating Zones using Pool with {cpu_cores} CPUs. Please wait...") + if verbose: + print(f" : Parallel creating Zones using Pool with {cpu_cores} CPUs. Please wait...") zone_dict_final = {} with Pool(cpu_cores) as pool: @@ -307,16 +327,20 @@ def read_zone(zone_file: str = "", cpu_cores: int = 1) -> dict[int: Zone]: for zone_dict in results: zone_dict_final.update(zone_dict) - print(f" : Successfully loaded zone.csv: {len(zone_dict_final)} Zones loaded.") + if verbose: + print(f" : Successfully loaded zone.csv: {len(zone_dict_final)} Zones loaded.") + return zone_dict_final @func_running_time -def read_network(input_folder: str = "", cpu_cores: int = 1) -> dict[str: dict]: +def read_network(input_folder: str = "", cpu_cores: int = 1, verbose: bool = False) -> dict[str: dict]: """Read node.csv and poi.csv files and return a dict of nodes and a dict of POIs. Args: input_folder (str, optional): required files within this folder. Defaults to current folder. + cpu_cores (int, optional): number of cpu cores for parallel processing. Defaults to 1. + verbose (bool, optional): print processing information. Defaults to False. Raises: FileNotFoundError: if input_folder does not exist. @@ -360,8 +384,10 @@ def read_network(input_folder: str = "", cpu_cores: int = 1) -> dict[str: dict]: if not is_required_files_exist: raise FileNotFoundError(f"Required files: {pkg_settings['required_files']} are not satisfied, please check your input folder.") - node_dict = read_node(input_folder + "/node.csv", cpu_cores) - poi_dict = read_poi(input_folder + "/poi.csv", cpu_cores) + node_dict = read_node(input_folder + "/node.csv", cpu_cores, verbose=verbose) + poi_dict = read_poi(input_folder + "/poi.csv", cpu_cores, verbose=verbose) + + if verbose: + print(f" : Successfully loaded node.csv and poi.csv: {len(node_dict)} Nodes and {len(poi_dict)} POIs.") - print(f" : Successfully loaded node.csv and poi.csv: {len(node_dict)} Nodes and {len(poi_dict)} POIs.") return {"node_dict": node_dict, "poi_dict": poi_dict} diff --git a/grid2demand/func_lib/trip_rate_production_attraction.py b/grid2demand/func_lib/trip_rate_production_attraction.py index 43a7d77..999c295 100644 --- a/grid2demand/func_lib/trip_rate_production_attraction.py +++ b/grid2demand/func_lib/trip_rate_production_attraction.py @@ -14,7 +14,8 @@ def gen_poi_trip_rate(poi_dict: dict, trip_rate_file: str = "", - trip_purpose: int = 1) -> dict: + trip_purpose: int = 1, + verbose: bool = False) -> dict: """Generate trip rate for each poi. Args: @@ -38,7 +39,7 @@ def gen_poi_trip_rate(poi_dict: dict, # if no poi trip rate file provided, use default trip rate if not trip_rate_file: - print("No trip rate file is provided, use default trip rate.") + print(" No trip rate file provided, use default trip rate.") default_flag = True # if trip rate file provided is not valid, use default trip rate elif not os.path.isfile(path2linux(trip_rate_file)): @@ -88,7 +89,10 @@ def gen_poi_trip_rate(poi_dict: dict, poi_type = poi_dict[poi_id].poi_type if poi_type in df_trip_rate_dict: poi_dict[poi_id].trip_rate = df_trip_rate_dict[poi_type].to_dict() - print(f" : Successfully generated poi trip rate from {trip_rate_file}.") + + if verbose: + print(f" : Successfully generated poi trip rate from {trip_rate_file}.") + return poi_dict @@ -97,7 +101,8 @@ def gen_node_prod_attr(node_dict: dict, residential_production: float = 10.0, residential_attraction: float = 10.0, boundary_production: float = 1000.0, - boundary_attraction: float = 1000.0) -> dict: + boundary_attraction: float = 1000.0, + verbose: bool = False) -> dict: """Generate production and attraction for each node. Args: @@ -139,5 +144,7 @@ def gen_node_prod_attr(node_dict: dict, else: node.production = 0 node.attraction = 0 - print(" : Successfully generated production and attraction for each node based on poi trip rate.") + if verbose: + print(" : Successfully generated production and attraction for each node based on poi trip rate.") + return node_dict diff --git a/grid2demand_tutorial_use_zone_in_nodeCSV.py b/grid2demand_tutorial_use_zone_in_nodeCSV.py index ff6b96e..467db6a 100644 --- a/grid2demand_tutorial_use_zone_in_nodeCSV.py +++ b/grid2demand_tutorial_use_zone_in_nodeCSV.py @@ -14,16 +14,16 @@ input_dir = "./datasets/dubai" # Initialize a GRID2DEMAND object - gd = GRID2DEMAND(input_dir) + gd = GRID2DEMAND(input_dir, use_zone_id=True) # Step 1: Load node and poi data from input directory node_dict, poi_dict = gd.load_network.values() # Step 2: Generate zone dictionary from node dictionary # by specifying number of x blocks and y blocks - zone_dict = gd.net2zone(node_dict, num_x_blocks=10, num_y_blocks=10, use_zone_id=True) + zone_dict = gd.net2zone(node_dict, num_x_blocks=10, num_y_blocks=10) - # Step 2: Generate zone based on grid size with 10 km width and 10km height for each zone + # by specifying cell_width and cell_height in km # zone_dict = gd.net2zone(node_dict, cell_width=10, cell_height=10, use_zone_id=True) # Step 3: synchronize geometry info between zone, node and poi @@ -45,10 +45,10 @@ zone_prod_attr = gd.calc_zone_prod_attr(node_prod_attr, zone_dict_update) # Step 7: Run gravity model to generate agent-based demand - df_demand = gd.run_gravity_model(zone_prod_attr, zone_od_dist_matrix, use_zone_id=True) + df_demand = gd.run_gravity_model(zone_prod_attr, zone_od_dist_matrix) # Step 8: generate agent-based demand - df_agent = gd.gen_agent_based_demand(node_prod_attr, zone_prod_attr, df_demand=df_demand) + df_agent = gd.gen_agent_based_demand(node_prod_attr, zone_prod_attr, df_demand=gd.df_demand) # You can also view and edit the package setting by using gd.pkg_settings print(gd.pkg_settings) @@ -58,4 +58,5 @@ gd.save_agent gd.save_zone gd.save_zone_od_dist_table - gd.save_zone_od_dist_matrix \ No newline at end of file + gd.save_zone_od_dist_matrix + gd.save_node_within_zone \ No newline at end of file