From c041c82e566f33a518be69d7918596d28e320a03 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Tue, 22 Oct 2024 16:51:46 +0200 Subject: [PATCH 1/7] Remove path setting in config --- message_ix_buildings/chilled/util/config.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/message_ix_buildings/chilled/util/config.py b/message_ix_buildings/chilled/util/config.py index f8ba6cb..bafe7ff 100644 --- a/message_ix_buildings/chilled/util/config.py +++ b/message_ix_buildings/chilled/util/config.py @@ -4,8 +4,6 @@ import numpy as np -from message_ix_buildings.chilled.functions.user_settings import DICT_USER_SETTINGS - @dataclass class Config: @@ -128,14 +126,6 @@ class Config: #: TODO: In the future, support "R12". node: Literal["R11"] = "R11" - #: Paths settings by user - project_path: str = str(DICT_USER_SETTINGS[user]["project_path"]) - dle_path: str = str(DICT_USER_SETTINGS[user]["dle_path"]) - message_region_file: str = str(DICT_USER_SETTINGS[user]["message_region_map_file"]) - isimip_bias_adj_path: str = str(DICT_USER_SETTINGS[user]["isimip_bias_adj_path"]) - isimip_ewemib_path: str = str(DICT_USER_SETTINGS[user]["isimip_ewembi_path"]) - chunk_size = DICT_USER_SETTINGS[user]["chunk_size"] - #: NetCDF settings netcdf4_format = "NETCDF4_CLASSIC" comp = dict(zlib=True, complevel=5) # Compression between 0 and 9 (highest) From 884681dc64b898ce12b02d0e59864e7a5f489947 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Tue, 22 Oct 2024 16:51:57 +0200 Subject: [PATCH 2/7] Add function to set paths based on user --- message_ix_buildings/chilled/util/util.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/message_ix_buildings/chilled/util/util.py b/message_ix_buildings/chilled/util/util.py index e3724e4..af923aa 100644 --- a/message_ix_buildings/chilled/util/util.py +++ b/message_ix_buildings/chilled/util/util.py @@ -3,13 +3,24 @@ from pathlib import Path import pandas as pd -from util.config import Config # type: ignore + +from message_ix_buildings.chilled.functions.user_settings import DICT_USER_SETTINGS +from message_ix_buildings.chilled.util.config import Config # type: ignore def get_project_root() -> Path: return Path(__file__).parent.parent.parent +def get_paths(config: "Config", selection): + # if selection is chunk_size, then return chunk_size as is + # else, return string version of the path + if selection == "chunk_size": + return DICT_USER_SETTINGS[config.user][selection] + else: + return str(DICT_USER_SETTINGS[config.user][selection]) + + def get_logger(name: str): log = logging.getLogger(name) log.setLevel(logging.INFO) From 9497a7843fd481757febf58ddb566cd0583e0736 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Tue, 22 Oct 2024 16:52:06 +0200 Subject: [PATCH 3/7] Update path setting in other scripts --- message_ix_buildings/chilled/core/climate.py | 68 +++++++++++++------ .../chilled/postprocess/postprocess.py | 7 +- .../chilled/preprocess/archetypes.py | 9 ++- 3 files changed, 57 insertions(+), 27 deletions(-) diff --git a/message_ix_buildings/chilled/core/climate.py b/message_ix_buildings/chilled/core/climate.py index 94cdd4c..daf0dfe 100644 --- a/message_ix_buildings/chilled/core/climate.py +++ b/message_ix_buildings/chilled/core/climate.py @@ -48,6 +48,7 @@ from message_ix_buildings.chilled.util.util import ( get_archs, get_logger, + get_paths, load_all_scenarios_data, load_parametric_analysis_data, ) @@ -56,7 +57,11 @@ def create_climate_variables_maps(config: "Config", start_time: datetime.datetime): - out_path = os.path.join(config.project_path, "out", "version", config.vstr) + project_path = get_paths(config, "project_path") + dle_path = get_paths(config, "dle_path") + isimip_bias_adj_path = get_paths(config, "isimip_bias_adj_path") + + out_path = os.path.join(project_path, "out", "version", config.vstr) archetype_path = os.path.join(out_path, "rasters") save_path = os.path.join(out_path, "VDD_ene_calcs") @@ -105,10 +110,10 @@ def map_calculated_variables(args): endstr = ".nc" if str(clim) == "hist": - isi_folder = config.isimip_ewemib_path + isi_folder = isimip_ewemib_path filestr = climate_filestr_hist else: - isi_folder = config.isimip_bias_adj_path + isi_folder = isimip_bias_adj_path filestr = climate_filestr_future filepath = os.path.join( @@ -135,12 +140,12 @@ def map_calculated_variables(args): t_oa_gbm = t_out_ave.groupby("time.month") i_sol_v = xr.open_dataarray( - os.path.join(config.dle_path, "EWEMBI_vert_irrad_1980-2009_avg.nc") + os.path.join(dle_path, "EWEMBI_vert_irrad_1980-2009_avg.nc") ) # Values in daily Wh/m2 # Horizontal irradiation i_sol_h = xr.open_dataarray( - os.path.join(config.dle_path, "EWEMBI_horiz_irrad_1980-2009_avg.nc") + os.path.join(dle_path, "EWEMBI_horiz_irrad_1980-2009_avg.nc") ) # Values in daily Wh/m2 if config.arch_setting == "regional": @@ -756,7 +761,8 @@ def read_netcdf_files(input_args): def aggregate_urban_rural_files(config: "Config"): - out_path = os.path.join(config.project_path, "out", "version", config.vstr) + project_path = get_paths(config, "project_path") + out_path = os.path.join(project_path, "out", "version", config.vstr) save_path = os.path.join(out_path, "VDD_ene_calcs") output_path_vdd = os.path.join( @@ -834,7 +840,8 @@ def aggregate_urban_rural_files(config: "Config"): def make_vdd_total_maps(config: "Config"): - out_path = os.path.join(config.project_path, "out", "version", config.vstr) + project_path = get_paths(config, "project_path") + out_path = os.path.join(project_path, "out", "version", config.vstr) save_path = os.path.join(out_path, "VDD_ene_calcs") output_path_vdd = os.path.join( @@ -1133,7 +1140,9 @@ def make_map( def process_construction_shares(config: "Config"): - out_path = os.path.join(config.project_path, "out", "version", config.vstr) + project_path = get_paths(config, "project_path") + dle_path = get_paths(config, "dle_path") + out_path = os.path.join(project_path, "out", "version", config.vstr) floorarea_path = os.path.join(out_path, "floorarea_country") output_path = os.path.join( @@ -1153,7 +1162,7 @@ def process_construction_shares(config: "Config"): # If constr_setting == 1, then process construction shares. Otherwise, skip if config.constr_setting == 1: - input_path = config.dle_path + input_path = dle_path dsc = xr.Dataset() for urt in config.urts: @@ -1226,8 +1235,10 @@ def process_construction_shares(config: "Config"): def process_floor_area_maps(config: "Config"): - input_path = config.dle_path - out_path = os.path.join(config.project_path, "out", "version", config.vstr) + project_path = get_paths(config, "project_path") + dle_path = get_paths(config, "dle_path") + input_path = dle_path + out_path = os.path.join(project_path, "out", "version", config.vstr) save_path = os.path.join(out_path, "floorarea_country") output_path = os.path.join( @@ -1354,8 +1365,11 @@ def process_floor_area_maps(config: "Config"): def process_country_maps(config: "Config"): - input_path = config.dle_path - out_path = os.path.join(config.project_path, "out", "version", config.vstr) + project_path = get_paths(config, "project_path") + dle_path = get_paths(config, "dle_path") + + input_path = dle_path + out_path = os.path.join(project_path, "out", "version", config.vstr) save_path = os.path.join(out_path, "floorarea_country") output_path = os.path.join( @@ -1436,8 +1450,11 @@ def process_country_maps(config: "Config"): def process_final_maps(config: "Config"): - input_path = config.dle_path - out_path = os.path.join(config.project_path, "out", "version", config.vstr) + project_path = get_paths(config, "project_path") + dle_path = get_paths(config, "dle_path") + + input_path = dle_path + out_path = os.path.join(project_path, "out", "version", config.vstr) vdd_path = os.path.join( out_path, "VDD_ene_calcs", @@ -1835,8 +1852,11 @@ def process_final_maps(config: "Config"): def process_iso_tables(config: "Config"): start = datetime.datetime.now() - input_path = config.dle_path - out_path = os.path.join(config.project_path, "out", "version", config.vstr) + project_path = get_paths(config, "project_path") + dle_path = get_paths(config, "dle_path") + + input_path = dle_path + out_path = os.path.join(project_path, "out", "version", config.vstr) vdd_path = os.path.join(out_path, "VDD_ene_calcs", config.gcm, config.rcp) floorarea_path = os.path.join(out_path, "floorarea_country", config.gcm, config.rcp) finalmaps_path = os.path.join(out_path, "final_maps", config.gcm, config.rcp) @@ -2140,7 +2160,11 @@ def aggregate_ncfile(args: tuple) -> xr.Dataset: def create_climate_outputs(config: "Config", start_time: datetime.datetime): - out_path = os.path.join(config.project_path, "out", "version", config.vstr) + project_path = get_paths(config, "project_path") + dle_path = get_paths(config, "dle_path") + isimip_bias_adj_path = get_paths(config, "isimip_bias_adj_path") + isimip_ewemib_path = get_paths(config, "isimip_ewemib_path") + out_path = os.path.join(project_path, "out", "version", config.vstr) archetype_path = os.path.join(out_path, "rasters") save_path = os.path.join(out_path, "VDD_ene_calcs") @@ -2180,10 +2204,10 @@ def create_climate_outputs(config: "Config", start_time: datetime.datetime): nyrs_clim = int(years_clim[1]) - int(years_clim[0]) + 1 if str(clim) == "hist": - isi_folder = config.isimip_ewemib_path + isi_folder = isimip_ewemib_path filestr = config.climate_filestr_hist else: - isi_folder = config.isimip_bias_adj_path + isi_folder = isimip_bias_adj_path filestr = config.climate_filestr_future filepath = os.path.join( @@ -2226,12 +2250,12 @@ def create_climate_outputs(config: "Config", start_time: datetime.datetime): # Vertical irradiation # i_sol_v = xr.open_dataarray(input_folder+'CERES_vert_irrad_2001-13_avg.nc') #Values in daily Wh/m2 i_sol_v = xr.open_dataarray( - os.path.join(config.dle_path, "EWEMBI_vert_irrad_1980-2009_avg.nc") + os.path.join(dle_path, "EWEMBI_vert_irrad_1980-2009_avg.nc") ) # Values in daily Wh/m2 # Horizontal irradiation i_sol_h = xr.open_dataarray( - os.path.join(config.dle_path, "EWEMBI_horiz_irrad_1980-2009_avg.nc") + os.path.join(dle_path, "EWEMBI_horiz_irrad_1980-2009_avg.nc") ) # Values in daily Wh/m2 # i_sol = i_sol.sel(time=slice(years_clim[0],years_clim[1])) diff --git a/message_ix_buildings/chilled/postprocess/postprocess.py b/message_ix_buildings/chilled/postprocess/postprocess.py index fd4316f..bef248b 100644 --- a/message_ix_buildings/chilled/postprocess/postprocess.py +++ b/message_ix_buildings/chilled/postprocess/postprocess.py @@ -3,10 +3,11 @@ import pandas as pd from message_ix_buildings.chilled.util.config import Config -from message_ix_buildings.chilled.util.util import get_logger +from message_ix_buildings.chilled.util.util import get_logger, get_paths log = get_logger(__name__) -cfg = Config() +cfg = Config(user="MEAS") +dle_path = get_paths(cfg, "dle_path") def get_sturm_data(input_path, input_version_name): @@ -254,4 +255,4 @@ def postprocess_electricity_demand(input_path, input_version_name): ) -postprocess_electricity_demand(cfg.dle_path, cfg.vstr) +postprocess_electricity_demand(dle_path, cfg.vstr) diff --git a/message_ix_buildings/chilled/preprocess/archetypes.py b/message_ix_buildings/chilled/preprocess/archetypes.py index aeaf55d..a37c8c8 100644 --- a/message_ix_buildings/chilled/preprocess/archetypes.py +++ b/message_ix_buildings/chilled/preprocess/archetypes.py @@ -15,6 +15,7 @@ from message_ix_buildings.chilled.util.util import ( get_archs, get_logger, + get_paths, read_arch_inputs_df, read_arch_reg_df, ) @@ -23,7 +24,9 @@ def create_archetypes(config: "Config"): - out_path = os.path.join(config.project_path, "out", "version") + project_path = get_paths(config, "dle_path") + + out_path = os.path.join(project_path, "out", "version") archetype_path = os.path.join(out_path, config.vstr, "rasters") # if archetypes folder does not exist, create it @@ -101,7 +104,9 @@ def create_archetypes(config: "Config"): def create_archetype_variables(config: "Config"): - out_path = os.path.join(config.project_path, "out", "version") + project_path = get_paths(config, "dle_path") + + out_path = os.path.join(project_path, "out", "version") archetype_path = os.path.join(out_path, config.vstr, "rasters") # get archs From b728bf37892b23c8da499dfb05826fabfd8633d4 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Tue, 22 Oct 2024 16:53:24 +0200 Subject: [PATCH 4/7] Add second version string to correct data path Currently data is stored in a folder that has a version number on it. It would be better to remove this so the files being read won't be dependent on this numbering system but instead should only depend on version name. --- message_ix_buildings/chilled/postprocess/postprocess.py | 2 +- message_ix_buildings/chilled/util/config.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/message_ix_buildings/chilled/postprocess/postprocess.py b/message_ix_buildings/chilled/postprocess/postprocess.py index bef248b..5ceabdf 100644 --- a/message_ix_buildings/chilled/postprocess/postprocess.py +++ b/message_ix_buildings/chilled/postprocess/postprocess.py @@ -255,4 +255,4 @@ def postprocess_electricity_demand(input_path, input_version_name): ) -postprocess_electricity_demand(dle_path, cfg.vstr) +postprocess_electricity_demand(dle_path, cfg.vstr2 + "_" + cfg.vstr) diff --git a/message_ix_buildings/chilled/util/config.py b/message_ix_buildings/chilled/util/config.py index bafe7ff..cac5d17 100644 --- a/message_ix_buildings/chilled/util/config.py +++ b/message_ix_buildings/chilled/util/config.py @@ -18,6 +18,7 @@ class Config: #: #: This is used to name the output files and directories. vstr: str = "ALPS2023" + vstr2 = "v19" #: Select the climate model. #: From 4e78f3fef88a867378bfe95f0545aedfb0f0881c Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Tue, 22 Oct 2024 17:01:13 +0200 Subject: [PATCH 5/7] Fix path name typo --- message_ix_buildings/chilled/core/climate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/message_ix_buildings/chilled/core/climate.py b/message_ix_buildings/chilled/core/climate.py index daf0dfe..45bb9bd 100644 --- a/message_ix_buildings/chilled/core/climate.py +++ b/message_ix_buildings/chilled/core/climate.py @@ -60,6 +60,7 @@ def create_climate_variables_maps(config: "Config", start_time: datetime.datetim project_path = get_paths(config, "project_path") dle_path = get_paths(config, "dle_path") isimip_bias_adj_path = get_paths(config, "isimip_bias_adj_path") + isimip_ewembi_path = get_paths(config, "isimip_ewembi_path") out_path = os.path.join(project_path, "out", "version", config.vstr) archetype_path = os.path.join(out_path, "rasters") @@ -110,7 +111,7 @@ def map_calculated_variables(args): endstr = ".nc" if str(clim) == "hist": - isi_folder = isimip_ewemib_path + isi_folder = isimip_ewembi_path filestr = climate_filestr_hist else: isi_folder = isimip_bias_adj_path From 4595c5cbe83ab9c42d1a67daed4e649c04c24e70 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Thu, 31 Oct 2024 13:53:13 +0100 Subject: [PATCH 6/7] Update paths to use `get_paths()` in preprocessing --- message_ix_buildings/chilled/preprocess/message_raster.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/message_ix_buildings/chilled/preprocess/message_raster.py b/message_ix_buildings/chilled/preprocess/message_raster.py index 60b3c17..e969dfe 100644 --- a/message_ix_buildings/chilled/preprocess/message_raster.py +++ b/message_ix_buildings/chilled/preprocess/message_raster.py @@ -10,6 +10,7 @@ import xarray as xr from message_ix_buildings.chilled.util.config import Config +from message_ix_buildings.chilled.util.util import get_paths def create_message_raster(config: "Config"): @@ -35,11 +36,12 @@ def create_message_raster(config: "Config"): ISO attributes in a dataframe """ - input_path = os.path.join(config.dle_path) + input_path = get_paths(config, "dle_path") + message_region_file = get_paths(config, "message_region_map_file") if config.node == "R11": msgregions = pd.read_excel( - config.message_region_file, + message_region_file, sheet_name="regional definition", ) From 96da133cb6ca45f7b96e73485122fbcb7ccdbf25 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Tue, 22 Oct 2024 17:19:08 +0200 Subject: [PATCH 7/7] Save postprocessed CHILLED data --- message_ix_buildings/chilled/postprocess/postprocess.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/message_ix_buildings/chilled/postprocess/postprocess.py b/message_ix_buildings/chilled/postprocess/postprocess.py index 5ceabdf..08d5c4b 100644 --- a/message_ix_buildings/chilled/postprocess/postprocess.py +++ b/message_ix_buildings/chilled/postprocess/postprocess.py @@ -197,6 +197,14 @@ def postprocess_electricity_demand(input_path, input_version_name): ) # Save files + + df_chilled.to_csv( + os.path.join(version_output_path, "chilled_postprocessed.csv"), + index=False, + ) + + log.info("Saved: " + os.path.join(version_output_path, "chilled_postprocessed.csv")) + df_sturm.to_csv( os.path.join(version_output_path, "sturm_building_stock_inputs.csv"), index=False,