From 8c03cd0a3a83904df60cd3ae9378b66ae4414a12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Theresa=20Gro=C3=9F?= Date: Tue, 16 Apr 2024 15:07:44 +0200 Subject: [PATCH 01/17] Add 2dim as parameter and delete doubled method --- fine/transmission.py | 7 ++++--- fine/utils.py | 28 ---------------------------- 2 files changed, 4 insertions(+), 31 deletions(-) diff --git a/fine/transmission.py b/fine/transmission.py index 7e4230a8..0829d100 100644 --- a/fine/transmission.py +++ b/fine/transmission.py @@ -408,16 +408,17 @@ def __init__( # operationRateMax self.operationRateMax = operationRateMax + print(operationRateMax) self.fullOperationRateMax = utils.checkAndSetInvestmentPeriodTimeSeries( - esM, name, operationRateMax, self.locationalEligibility + esM, name, operationRateMax, self.locationalEligibility, "2dim" ) self.aggregatedOperationRateMax = dict.fromkeys(esM.investmentPeriods) self.processedOperationRateMax = dict.fromkeys(esM.investmentPeriods) - + print(operationRateFix) # operationRateFix self.operationRateFix = operationRateFix self.fullOperationRateFix = utils.checkAndSetInvestmentPeriodTimeSeries( - esM, name, operationRateFix, self.locationalEligibility + esM, name, operationRateFix, self.locationalEligibility, "2dim" ) self.aggregatedOperationRateFix = dict.fromkeys(esM.investmentPeriods) self.processedOperationRateFix = dict.fromkeys(esM.investmentPeriods) diff --git a/fine/utils.py b/fine/utils.py index 83511b99..a9ff226f 100644 --- a/fine/utils.py +++ b/fine/utils.py @@ -1064,33 +1064,6 @@ def checkAndSetInvestmentPeriodTimeSeries( return parameter -def checkAndSetInvestmentPeriodTimeSeries( - esM, name, data, locationalEligibility, dimension="1dim" -): - checkInvestmentPeriodParameters(name, data, esM.investmentPeriodNames) - parameter = {} - for _ip in esM.investmentPeriodNames: - # map name of investment period (e.g. 2020) to index (e.g. 0) - ip = esM.investmentPeriodNames.index(_ip) - if ( - isinstance(data, pd.DataFrame) - or data is None - or isinstance(data, pd.Series) - ): - parameter[ip] = checkAndSetTimeSeries( - esM, name, data, locationalEligibility, dimension - ) - elif isinstance(data, dict): - parameter[ip] = checkAndSetTimeSeries( - esM, name, data[_ip], locationalEligibility, dimension - ) - else: - raise TypeError( - f"Parameter of {name} should be a pandas dataframe or a dictionary." - ) - return parameter - - def checkAndSetTimeSeries( esM, name, operationTimeSeries, locationalEligibility, dimension="1dim" ): @@ -1202,7 +1175,6 @@ def checkAndSetTimeSeries( else: return None - def checkDesignVariableModelingParameters( esM, capacityVariableDomain, From a2b221f1eaefb62b18856eea0e882dc2c130fddc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Theresa=20Gro=C3=9F?= Date: Tue, 16 Apr 2024 15:46:07 +0200 Subject: [PATCH 02/17] Delete print calls --- fine/transmission.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fine/transmission.py b/fine/transmission.py index 0829d100..54d67bdb 100644 --- a/fine/transmission.py +++ b/fine/transmission.py @@ -408,13 +408,12 @@ def __init__( # operationRateMax self.operationRateMax = operationRateMax - print(operationRateMax) self.fullOperationRateMax = utils.checkAndSetInvestmentPeriodTimeSeries( esM, name, operationRateMax, self.locationalEligibility, "2dim" ) self.aggregatedOperationRateMax = dict.fromkeys(esM.investmentPeriods) self.processedOperationRateMax = dict.fromkeys(esM.investmentPeriods) - print(operationRateFix) + # operationRateFix self.operationRateFix = operationRateFix self.fullOperationRateFix = utils.checkAndSetInvestmentPeriodTimeSeries( From 0767acbd1c3f66e416c5d351e021091ecdd629dc Mon Sep 17 00:00:00 2001 From: "t.busch" Date: Wed, 17 Apr 2024 15:59:04 +0200 Subject: [PATCH 03/17] fix issue #338 --- fine/energySystemModel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fine/energySystemModel.py b/fine/energySystemModel.py index 81312d3c..ec9046b1 100644 --- a/fine/energySystemModel.py +++ b/fine/energySystemModel.py @@ -2031,7 +2031,7 @@ def optimize( ################################################################################################################ # Set which solver should solve the specified optimization problem - optimizer = opt.SolverFactory(solver) + optimizer = opt.SolverFactory(solver, solver_io="python") # Set, if specified, the time limit if self.solverSpecs["timeLimit"] is not None and solver == "gurobi": From 2d76e705e95e3fb027379e988be40b7633020093 Mon Sep 17 00:00:00 2001 From: "t.busch" Date: Wed, 17 Apr 2024 16:14:22 +0200 Subject: [PATCH 04/17] add different cases for solver_io --- fine/energySystemModel.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fine/energySystemModel.py b/fine/energySystemModel.py index ec9046b1..4c61ae1e 100644 --- a/fine/energySystemModel.py +++ b/fine/energySystemModel.py @@ -2031,7 +2031,10 @@ def optimize( ################################################################################################################ # Set which solver should solve the specified optimization problem - optimizer = opt.SolverFactory(solver, solver_io="python") + if solver == "gurobi": + optimizer = opt.SolverFactory(solver, solver_io="python") + else: + optimizer = opt.SolverFactory(solver) # Set, if specified, the time limit if self.solverSpecs["timeLimit"] is not None and solver == "gurobi": From e0984d53e0900d2d92ae18a74a4d98c068c35def Mon Sep 17 00:00:00 2001 From: Kevin Knosala Date: Thu, 18 Apr 2024 10:50:37 +0200 Subject: [PATCH 05/17] Check if gurobipy is installed for gurobi python api --- fine/energySystemModel.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fine/energySystemModel.py b/fine/energySystemModel.py index 4c61ae1e..8907d64d 100644 --- a/fine/energySystemModel.py +++ b/fine/energySystemModel.py @@ -2,6 +2,7 @@ import os import time import warnings +import importlib.util import gurobi_logtools as glt import pandas as pd @@ -2031,7 +2032,8 @@ def optimize( ################################################################################################################ # Set which solver should solve the specified optimization problem - if solver == "gurobi": + if solver == "gurobi" and importlib.util.find_spec('gurobipy'): + # Use the direct gurobi solver that uses the Python API. optimizer = opt.SolverFactory(solver, solver_io="python") else: optimizer = opt.SolverFactory(solver) From f73f5b2f80168b159de835517c27f4cd10533f91 Mon Sep 17 00:00:00 2001 From: "p.dunkel" Date: Thu, 18 Apr 2024 13:43:57 +0200 Subject: [PATCH 06/17] add test for xrds transmisison input --- test/IOManagement/test_xarrayio.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/IOManagement/test_xarrayio.py b/test/IOManagement/test_xarrayio.py index dea44b68..2fa0d2f6 100644 --- a/test/IOManagement/test_xarrayio.py +++ b/test/IOManagement/test_xarrayio.py @@ -240,3 +240,18 @@ def test_esm_to_datasets_with_processed_values(minimal_test_esM): .item() == 0.177 ) + +def test_transmission_dims(minimal_test_esM): + esM = minimal_test_esM + capacityMin=DataFrame([[0, 1], [1, 0]], index=list(esM.locations), columns=list(esM.locations)) + + # update Pipeline component + esM.updateComponent( + componentName="Pipelines", + updateAttrs={"capacityMin": capacityMin}, + ) + esM.optimize() + xr_dss = xrIO.convertOptimizationInputToDatasets(esM) + assert (esM.totalTimeSteps == list(xr_dss["Input"]["Transmission"]["Pipelines"].time.to_numpy())) + + From 9a1fbe41a6234a32394f7dbd9acc35f9de8e7d64 Mon Sep 17 00:00:00 2001 From: "p.dunkel" Date: Thu, 18 Apr 2024 14:00:25 +0200 Subject: [PATCH 07/17] add test to check xrds to esM conversion --- test/IOManagement/test_xarrayio.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/IOManagement/test_xarrayio.py b/test/IOManagement/test_xarrayio.py index 2fa0d2f6..5db6658d 100644 --- a/test/IOManagement/test_xarrayio.py +++ b/test/IOManagement/test_xarrayio.py @@ -254,4 +254,7 @@ def test_transmission_dims(minimal_test_esM): xr_dss = xrIO.convertOptimizationInputToDatasets(esM) assert (esM.totalTimeSteps == list(xr_dss["Input"]["Transmission"]["Pipelines"].time.to_numpy())) + esM2 = xrIO.convertDatasetsToEnergySystemModel(xrds) + capacityMin = esM2.getComponentAttribute("Pipelines", "capacityMin") + assert capacityMin.index.name == "space" From a5e885d8a74dc9f071fa054c48c2e32dadf7249e Mon Sep 17 00:00:00 2001 From: "p.dunkel" Date: Thu, 18 Apr 2024 14:03:53 +0200 Subject: [PATCH 08/17] test operationRate for transmission xrds input --- test/IOManagement/test_xarrayio.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/IOManagement/test_xarrayio.py b/test/IOManagement/test_xarrayio.py index 5db6658d..b27b39a7 100644 --- a/test/IOManagement/test_xarrayio.py +++ b/test/IOManagement/test_xarrayio.py @@ -250,6 +250,18 @@ def test_transmission_dims(minimal_test_esM): componentName="Pipelines", updateAttrs={"capacityMin": capacityMin}, ) + + time_index = pd.date_range(start="2020-01-01", periods=4, freq="H") + _locs = pd.MultiIndex.from_product([["ElectrolyzerLocation"],["IndustryLocation"]]) + columns = [f"{idx0}_{idx1}" for idx0, idx1 in _locs] + column2 = [f"{idx1}_{idx0}" for idx0, idx1 in _locs] + columns = columns + column2 + operationRateMax = pd.DataFrame(1, index=time_index, columns=columns).reset_index(drop=True) + esM.updateComponent( + componentName="Pipelines", + updateAttrs={"operationRateMax": operationRateMax}, + ) + esM.optimize() xr_dss = xrIO.convertOptimizationInputToDatasets(esM) assert (esM.totalTimeSteps == list(xr_dss["Input"]["Transmission"]["Pipelines"].time.to_numpy())) From daf877717ebf7b2642d6f9a9c0000fbf9ce112f8 Mon Sep 17 00:00:00 2001 From: "p.dunkel" Date: Thu, 18 Apr 2024 14:05:23 +0200 Subject: [PATCH 09/17] fix imports --- test/IOManagement/test_xarrayio.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/IOManagement/test_xarrayio.py b/test/IOManagement/test_xarrayio.py index b27b39a7..f787c4da 100644 --- a/test/IOManagement/test_xarrayio.py +++ b/test/IOManagement/test_xarrayio.py @@ -5,7 +5,7 @@ import fine.IOManagement.xarrayIO as xrIO from fine.IOManagement.dictIO import exportToDict import fine as fn - +import pandas as pd def compare_values(value_1, value_2): """Apply assert functions from pandas if values are pandas.DataFrame or @@ -242,8 +242,9 @@ def test_esm_to_datasets_with_processed_values(minimal_test_esM): ) def test_transmission_dims(minimal_test_esM): + esM = minimal_test_esM - capacityMin=DataFrame([[0, 1], [1, 0]], index=list(esM.locations), columns=list(esM.locations)) + capacityMin=pd.DataFrame([[0, 1], [1, 0]], index=list(esM.locations), columns=list(esM.locations)) # update Pipeline component esM.updateComponent( From 7bf23b78c6fbd5557631970a24e0b64dd7f7804d Mon Sep 17 00:00:00 2001 From: "p.dunkel" Date: Thu, 18 Apr 2024 14:30:43 +0200 Subject: [PATCH 10/17] add seperate processing of transmission components in addDFVariablesToXarray --- fine/IOManagement/utilsIO.py | 80 ++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/fine/IOManagement/utilsIO.py b/fine/IOManagement/utilsIO.py index 994e8fdc..934876ae 100644 --- a/fine/IOManagement/utilsIO.py +++ b/fine/IOManagement/utilsIO.py @@ -240,7 +240,87 @@ def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict): :return: xr_ds """ + # Treat transmission data separately + df_iteration_dict_transm = {} + for variable_description, description_tuple_list in df_iteration_dict.copy().items(): + for description_tuple in description_tuple_list: + # check if data is transmission and time dependent + if "Transmission" in description_tuple[0]: + # add "2dim" to variable_description + df_iteration_dict_transm[variable_description] = [description_tuple] + # drop description_tuple from list + df_iteration_dict[variable_description].remove(description_tuple) + + for variable_description, description_tuple_list in df_iteration_dict_transm.items(): + df_dict = {} + + for description_tuple in description_tuple_list: + classname, component = description_tuple + + df_description = f"{classname}; {component}" + + # If a . is present in variable name, then the data would be + # another level further in the component_dict + if "." in variable_description: + [var_name, subvar_name] = variable_description.split(".") + if subvar_name.isdigit(): + subvar_name = int(subvar_name) + data = component_dict[classname][component][var_name][subvar_name] + else: + data = component_dict[classname][component][variable_description] + + multi_index_dataframe = data.stack() + if locations == set(component_dict[classname][component][variable_description].index.to_list()): + multi_index_dataframe.index.set_names("space", level=0, inplace=True) + multi_index_dataframe.index.set_names("space_2", level=1, inplace=True) + else: + # split X_X into multiindex + multi_index_dataframe.index.set_names("time", level=0, inplace=True) + multi_index_dataframe.index.set_names("space", level=1, inplace=True) + # use regex to split via location names + import re + pattern = re.compile("(" + "|".join(locations) + ")") + space_index = multi_index_dataframe.index.get_level_values("space").str.findall(pattern) + time_index = multi_index_dataframe.index.get_level_values("time") + # reconstruct multiindex + multi_index_dataframe.index = pd.MultiIndex.from_tuples( + [(time_index[i], space_index[i][0], space_index[i][1]) for i in range(len(space_index))], + names=["time", "space", "space_2"] + ) + + df_dict[df_description] = multi_index_dataframe + + df_variable = pd.concat(df_dict) + df_variable.index.set_names("component", level=0, inplace=True) + + ds_component = xr.Dataset() + if "time" in df_variable.index.names: + ds_component[f"ts_{variable_description}"] = ( + df_variable.sort_index().to_xarray() + ) + else: + ds_component[f"2d_{variable_description}"] = ( + df_variable.sort_index().to_xarray() + ) + + for comp in df_variable.index.get_level_values(0).unique(): + this_class = comp.split("; ")[0] + this_comp = comp.split("; ")[1] + + this_ds_component = ( + ds_component.sel(component=comp) + .squeeze() + .reset_coords(names=["component"], drop=True) + ) + try: + xr_ds[this_class][this_comp] = xr.merge( + [xr_ds[this_class][this_comp], this_ds_component] + ) + except Exception: + pass + + for variable_description, description_tuple_list in df_iteration_dict.items(): df_dict = {} From 0da420ed1b7ed101c20d35d3016828db6aae3b6c Mon Sep 17 00:00:00 2001 From: Kevin Knosala Date: Thu, 18 Apr 2024 13:43:47 +0200 Subject: [PATCH 11/17] Improve deploy --- .gitlab-ci.yml | 52 +++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4749dff8..9da764a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ image: mambaorg/micromamba stages: - test - - build + - deploy variables: DOCKER_HOST: tcp://docker:2375 @@ -38,6 +38,8 @@ variables: when: never - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS when: never + - if: $CI_COMMIT_TAG + when: never retry: 1 @@ -65,6 +67,8 @@ variables: - requirements.yml - requirements_dev.yml when: never + - if: $CI_COMMIT_TAG + when: never - when: on_success @@ -100,6 +104,8 @@ variables: when: never - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS when: never + - if: $CI_COMMIT_TAG + when: never - changes: - pyproject.toml - requirements.yml @@ -107,7 +113,7 @@ variables: when: on_success .build_template: - stage: build + stage: deploy image: docker@sha256:c8bb6fa5388b56304dd770c4bc0478de81ce18540173b1a589178c0d31bfce90 services: - docker:dind@sha256:c8bb6fa5388b56304dd770c4bc0478de81ce18540173b1a589178c0d31bfce90 @@ -121,6 +127,8 @@ test-pypi: - python -m pip install .[develop] - python -m pytest -n auto test/ rules: + - if: $CI_COMMIT_TAG + when: never - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH == "develop"' - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"' @@ -203,27 +211,12 @@ test-codestyle: # Deployment -build-master-latest: - extends: .build_template - script: - # Login to the DockerHub repo using a specialized access token. - # Then, build the docker image with the tested code and tag it - # with the current version, as well as latest. - # Afterwards, push to DockerHub. - - docker login -u fzjiek3 -p $DOCKER_AT - - docker build -t "fzjiek3/fine:latest" . - - docker push fzjiek3/fine:latest - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - when: never - - if: $CI_COMMIT_BRANCH == "master" - build-tag: extends: .build_template script: - docker login -u fzjiek3 -p $DOCKER_AT - - docker build -t fzjiek3/fine:${CI_COMMIT_TAG} . - - docker push fzjiek3/fine:${CI_COMMIT_TAG} + - docker build -t fzjiek3/fine:${CI_COMMIT_TAG} -t fzjiek3/fine:latest . + - docker push fzjiek3/fine --all-tags rules: - if: $CI_PIPELINE_SOURCE == "schedule" when: never @@ -242,14 +235,21 @@ build-dev: - if: $CI_COMMIT_BRANCH == "develop" pypi-upload: - stage: build + stage: deploy + image: python:3.12 + before_script: + - python3 -m pip install --upgrade build + - python3 -m pip install --upgrade twine + variables: + TWINE_USERNAME: $PYPI_USERNAME + TWINE_PASSWORD: $PYPI_PASSWORD script: - - micromamba install -c conda-forge -n base -y python=3.10 - - python -m pip install --upgrade build - - python -m pip install --upgrade twine - - python -m build - - python -m twine upload -u __token__ -p $PYPI_TOKEN dist/* + # Test if the version defined in `pyproject.toml` is the same as the tag + - PYPROJECT_VERSION=$(grep -m 1 version pyproject.toml | tr -s ' ' | tr -d '"' | tr -d "'" | cut -d' ' -f3) + - test PYPROJECT_VERSION = v${CI_COMMIT_TAG} + # Build and push to pypi + - python3 -m build + - python3 -m twine upload dist/* rules: - if: $CI_COMMIT_BRANCH == "master" && $CI_COMMIT_TAG - when: manual From a4faf13a93fc9d849e2d2ccada110adedf196092 Mon Sep 17 00:00:00 2001 From: "p.dunkel" Date: Thu, 18 Apr 2024 14:39:52 +0200 Subject: [PATCH 12/17] construct tranmission data in correct format for xarray --- fine/IOManagement/utilsIO.py | 49 +++++++++++++++++++---------------- fine/IOManagement/xarrayIO.py | 2 +- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/fine/IOManagement/utilsIO.py b/fine/IOManagement/utilsIO.py index 934876ae..484ea8e3 100644 --- a/fine/IOManagement/utilsIO.py +++ b/fine/IOManagement/utilsIO.py @@ -223,7 +223,7 @@ def generateIterationDicts(component_dict, investmentPeriods): return df_iteration_dict, series_iteration_dict, constants_iteration_dict -def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict): +def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict, locations): """Adds all variables whose data is contained in a pd.DataFrame to xarray dataset. These variables are normally regional time series (dimensions - space, time) @@ -237,6 +237,9 @@ def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict): keys - DF variable names values - list of tuple of component class and component name :type df_iteration_dict: dict + + :param locations: esM locations + :type locations: list :return: xr_ds """ @@ -270,7 +273,7 @@ def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict): data = component_dict[classname][component][variable_description] multi_index_dataframe = data.stack() - if locations == set(component_dict[classname][component][variable_description].index.to_list()): + if set(locations) == set(component_dict[classname][component][variable_description].index.to_list()): multi_index_dataframe.index.set_names("space", level=0, inplace=True) multi_index_dataframe.index.set_names("space_2", level=1, inplace=True) else: @@ -349,30 +352,32 @@ def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict): df_dict[df_description] = multi_index_dataframe - df_variable = pd.concat(df_dict) - df_variable.index.set_names("component", level=0, inplace=True) - - ds_component = xr.Dataset() - ds_component[f"ts_{variable_description}"] = ( - df_variable.sort_index().to_xarray() - ) - - for comp in df_variable.index.get_level_values(0).unique(): - this_class = comp.split("; ")[0] - this_comp = comp.split("; ")[1] + # check if there is data + if len(df_dict) > 0: + df_variable = pd.concat(df_dict) + df_variable.index.set_names("component", level=0, inplace=True) - this_ds_component = ( - ds_component.sel(component=comp) - .squeeze() - .reset_coords(names=["component"], drop=True) + ds_component = xr.Dataset() + ds_component[f"ts_{variable_description}"] = ( + df_variable.sort_index().to_xarray() ) - try: - xr_ds[this_class][this_comp] = xr.merge( - [xr_ds[this_class][this_comp], this_ds_component] + for comp in df_variable.index.get_level_values(0).unique(): + this_class = comp.split("; ")[0] + this_comp = comp.split("; ")[1] + + this_ds_component = ( + ds_component.sel(component=comp) + .squeeze() + .reset_coords(names=["component"], drop=True) ) - except Exception: - pass + + try: + xr_ds[this_class][this_comp] = xr.merge( + [xr_ds[this_class][this_comp], this_ds_component] + ) + except Exception: + pass return xr_ds diff --git a/fine/IOManagement/xarrayIO.py b/fine/IOManagement/xarrayIO.py index 0775a63a..d17a8084 100644 --- a/fine/IOManagement/xarrayIO.py +++ b/fine/IOManagement/xarrayIO.py @@ -46,7 +46,7 @@ def convertOptimizationInputToDatasets(esM, useProcessedValues=False): } # STEP 4. Add all df variables to xr_ds - xr_dss = utilsIO.addDFVariablesToXarray(xr_dss, component_dict, df_iteration_dict) + xr_dss = utilsIO.addDFVariablesToXarray(xr_dss, component_dict, df_iteration_dict, list(esM.locations)) # STEP 5. Add all series variables to xr_ds locations = sorted(esm_dict["locations"]) From 35ca4a12a2666ed37bd26ae97ff67c1afc5a2e6b Mon Sep 17 00:00:00 2001 From: "p.dunkel" Date: Thu, 18 Apr 2024 16:04:36 +0200 Subject: [PATCH 13/17] adapt test --- test/IOManagement/test_xarrayio.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/IOManagement/test_xarrayio.py b/test/IOManagement/test_xarrayio.py index f787c4da..20b28c9e 100644 --- a/test/IOManagement/test_xarrayio.py +++ b/test/IOManagement/test_xarrayio.py @@ -267,7 +267,9 @@ def test_transmission_dims(minimal_test_esM): xr_dss = xrIO.convertOptimizationInputToDatasets(esM) assert (esM.totalTimeSteps == list(xr_dss["Input"]["Transmission"]["Pipelines"].time.to_numpy())) - esM2 = xrIO.convertDatasetsToEnergySystemModel(xrds) + esM2 = xrIO.convertDatasetsToEnergySystemModel(xr_dss) - capacityMin = esM2.getComponentAttribute("Pipelines", "capacityMin") - assert capacityMin.index.name == "space" + operationRateMax = esM2.getComponentAttribute("Pipelines", "operationRateMax") + assert operationRateMax.index.name == "time" + + esM2.optimize() From c1b2dfa2131651fdcf2703336c2b01f20f760e49 Mon Sep 17 00:00:00 2001 From: "p.dunkel" Date: Thu, 18 Apr 2024 16:05:04 +0200 Subject: [PATCH 14/17] adapt addDFVariablesToXarray where transmisison components are now treated sepearately --- fine/IOManagement/utilsIO.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/fine/IOManagement/utilsIO.py b/fine/IOManagement/utilsIO.py index 484ea8e3..3a5b9239 100644 --- a/fine/IOManagement/utilsIO.py +++ b/fine/IOManagement/utilsIO.py @@ -244,15 +244,22 @@ def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict, locations): :return: xr_ds """ # Treat transmission data separately + df_iteration_dict_orig = df_iteration_dict.copy() df_iteration_dict_transm = {} - for variable_description, description_tuple_list in df_iteration_dict.copy().items(): + df_iteration_dict = {} + for variable_description, description_tuple_list in df_iteration_dict_orig.items(): for description_tuple in description_tuple_list: # check if data is transmission and time dependent if "Transmission" in description_tuple[0]: # add "2dim" to variable_description - df_iteration_dict_transm[variable_description] = [description_tuple] - # drop description_tuple from list - df_iteration_dict[variable_description].remove(description_tuple) + if variable_description not in df_iteration_dict_transm.keys(): + df_iteration_dict_transm[variable_description] = [] + df_iteration_dict_transm[variable_description].append(description_tuple) + + else: + if variable_description not in df_iteration_dict.keys(): + df_iteration_dict[variable_description] = [] + df_iteration_dict[variable_description].append(description_tuple) for variable_description, description_tuple_list in df_iteration_dict_transm.items(): df_dict = {} @@ -709,10 +716,22 @@ def addTimeSeriesVariableToDict( df = comp_var_xr.to_series() elif drop_component: df = comp_var_xr.drop("component").to_dataframe().unstack(level=1) + elif "space_2" in comp_var_xr.dims: + df = comp_var_xr.to_dataframe().squeeze() + # merge space and space_2 levels + space_index = df.index.get_level_values("space") + space_2_index = df.index.get_level_values("space_2") + new_space_index = [f"{space_index[i]}_{space_2_index[i]}" for i in range(len(space_index))] + df.index = pd.MultiIndex.from_tuples( + [(df.index.get_level_values("time")[i], new_space_index[i]) for i in range(len(new_space_index))], + names=["time", "space"] + ) + df = df.unstack() + df = df.dropna(axis=1, how="all") else: df = comp_var_xr.to_dataframe().unstack(level=1) - if isinstance(df, pd.DataFrame): + if isinstance(df, pd.DataFrame) and not "space_2" in comp_var_xr.dims: if len(df.columns) > 1: df.columns = df.columns.droplevel(0) From 2159751e3f4ce19522f8beaa8f49ff418e277486 Mon Sep 17 00:00:00 2001 From: "p.dunkel" Date: Thu, 18 Apr 2024 16:41:16 +0200 Subject: [PATCH 15/17] format --- fine/IOManagement/utilsIO.py | 41 +++++++++++++++++++++--------- test/IOManagement/test_xarrayio.py | 33 ++++++++++++++---------- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/fine/IOManagement/utilsIO.py b/fine/IOManagement/utilsIO.py index 3a5b9239..4ec21a9b 100644 --- a/fine/IOManagement/utilsIO.py +++ b/fine/IOManagement/utilsIO.py @@ -237,7 +237,7 @@ def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict, locations): keys - DF variable names values - list of tuple of component class and component name :type df_iteration_dict: dict - + :param locations: esM locations :type locations: list @@ -261,7 +261,10 @@ def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict, locations): df_iteration_dict[variable_description] = [] df_iteration_dict[variable_description].append(description_tuple) - for variable_description, description_tuple_list in df_iteration_dict_transm.items(): + for ( + variable_description, + description_tuple_list, + ) in df_iteration_dict_transm.items(): df_dict = {} for description_tuple in description_tuple_list: @@ -280,22 +283,32 @@ def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict, locations): data = component_dict[classname][component][variable_description] multi_index_dataframe = data.stack() - if set(locations) == set(component_dict[classname][component][variable_description].index.to_list()): + if set(locations) == set( + component_dict[classname][component][ + variable_description + ].index.to_list() + ): multi_index_dataframe.index.set_names("space", level=0, inplace=True) multi_index_dataframe.index.set_names("space_2", level=1, inplace=True) else: # split X_X into multiindex multi_index_dataframe.index.set_names("time", level=0, inplace=True) multi_index_dataframe.index.set_names("space", level=1, inplace=True) - # use regex to split via location names + # use regex to split via location names import re + pattern = re.compile("(" + "|".join(locations) + ")") - space_index = multi_index_dataframe.index.get_level_values("space").str.findall(pattern) + space_index = multi_index_dataframe.index.get_level_values( + "space" + ).str.findall(pattern) time_index = multi_index_dataframe.index.get_level_values("time") # reconstruct multiindex multi_index_dataframe.index = pd.MultiIndex.from_tuples( - [(time_index[i], space_index[i][0], space_index[i][1]) for i in range(len(space_index))], - names=["time", "space", "space_2"] + [ + (time_index[i], space_index[i][0], space_index[i][1]) + for i in range(len(space_index)) + ], + names=["time", "space", "space_2"], ) df_dict[df_description] = multi_index_dataframe @@ -329,8 +342,7 @@ def addDFVariablesToXarray(xr_ds, component_dict, df_iteration_dict, locations): ) except Exception: pass - - + for variable_description, description_tuple_list in df_iteration_dict.items(): df_dict = {} @@ -721,10 +733,15 @@ def addTimeSeriesVariableToDict( # merge space and space_2 levels space_index = df.index.get_level_values("space") space_2_index = df.index.get_level_values("space_2") - new_space_index = [f"{space_index[i]}_{space_2_index[i]}" for i in range(len(space_index))] + new_space_index = [ + f"{space_index[i]}_{space_2_index[i]}" for i in range(len(space_index)) + ] df.index = pd.MultiIndex.from_tuples( - [(df.index.get_level_values("time")[i], new_space_index[i]) for i in range(len(new_space_index))], - names=["time", "space"] + [ + (df.index.get_level_values("time")[i], new_space_index[i]) + for i in range(len(new_space_index)) + ], + names=["time", "space"], ) df = df.unstack() df = df.dropna(axis=1, how="all") diff --git a/test/IOManagement/test_xarrayio.py b/test/IOManagement/test_xarrayio.py index 20b28c9e..c0f86bea 100644 --- a/test/IOManagement/test_xarrayio.py +++ b/test/IOManagement/test_xarrayio.py @@ -7,6 +7,7 @@ import fine as fn import pandas as pd + def compare_values(value_1, value_2): """Apply assert functions from pandas if values are pandas.DataFrame or pandas.Series, else compare with `==` operator.""" @@ -241,35 +242,41 @@ def test_esm_to_datasets_with_processed_values(minimal_test_esM): == 0.177 ) + def test_transmission_dims(minimal_test_esM): - esM = minimal_test_esM - capacityMin=pd.DataFrame([[0, 1], [1, 0]], index=list(esM.locations), columns=list(esM.locations)) + capacityMin = pd.DataFrame( + [[0, 1], [1, 0]], index=list(esM.locations), columns=list(esM.locations) + ) # update Pipeline component esM.updateComponent( - componentName="Pipelines", - updateAttrs={"capacityMin": capacityMin}, - ) - + componentName="Pipelines", + updateAttrs={"capacityMin": capacityMin}, + ) + time_index = pd.date_range(start="2020-01-01", periods=4, freq="H") - _locs = pd.MultiIndex.from_product([["ElectrolyzerLocation"],["IndustryLocation"]]) + _locs = pd.MultiIndex.from_product([["ElectrolyzerLocation"], ["IndustryLocation"]]) columns = [f"{idx0}_{idx1}" for idx0, idx1 in _locs] column2 = [f"{idx1}_{idx0}" for idx0, idx1 in _locs] columns = columns + column2 - operationRateMax = pd.DataFrame(1, index=time_index, columns=columns).reset_index(drop=True) + operationRateMax = pd.DataFrame(1, index=time_index, columns=columns).reset_index( + drop=True + ) esM.updateComponent( - componentName="Pipelines", - updateAttrs={"operationRateMax": operationRateMax}, + componentName="Pipelines", + updateAttrs={"operationRateMax": operationRateMax}, ) - + esM.optimize() xr_dss = xrIO.convertOptimizationInputToDatasets(esM) - assert (esM.totalTimeSteps == list(xr_dss["Input"]["Transmission"]["Pipelines"].time.to_numpy())) + assert esM.totalTimeSteps == list( + xr_dss["Input"]["Transmission"]["Pipelines"].time.to_numpy() + ) esM2 = xrIO.convertDatasetsToEnergySystemModel(xr_dss) operationRateMax = esM2.getComponentAttribute("Pipelines", "operationRateMax") assert operationRateMax.index.name == "time" - + esM2.optimize() From 7c34e094aa28238531b349cd58de0fd237d4f142 Mon Sep 17 00:00:00 2001 From: Kevin Knosala Date: Thu, 18 Apr 2024 18:00:30 +0200 Subject: [PATCH 16/17] Fix linting --- fine/IOManagement/utilsIO.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fine/IOManagement/utilsIO.py b/fine/IOManagement/utilsIO.py index 4ec21a9b..e58be40c 100644 --- a/fine/IOManagement/utilsIO.py +++ b/fine/IOManagement/utilsIO.py @@ -748,7 +748,7 @@ def addTimeSeriesVariableToDict( else: df = comp_var_xr.to_dataframe().unstack(level=1) - if isinstance(df, pd.DataFrame) and not "space_2" in comp_var_xr.dims: + if isinstance(df, pd.DataFrame) and "space_2" not in comp_var_xr.dims: if len(df.columns) > 1: df.columns = df.columns.droplevel(0) From d940640217c125fa1809ea80ea4bce109e2e1a9f Mon Sep 17 00:00:00 2001 From: Kevin Knosala Date: Mon, 22 Apr 2024 09:49:20 +0200 Subject: [PATCH 17/17] Preparing v2.3.7 release --- docs/source/conf.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 87f32a38..2cc16a3b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -80,7 +80,7 @@ # The short X.Y version. version = "2.3" # The full version, including alpha/beta/rc tags. -release = "2.3.6" +release = "2.3.7" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pyproject.toml b/pyproject.toml index 95674070..54519ad8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "fine" -version = "2.3.6" +version = "2.3.7" description = "Framework for integrated energy systems assessment" readme = "README.md" authors = [