From deaf9bec4166b6eccb777f3588a04bb5e6665598 Mon Sep 17 00:00:00 2001 From: DH271725 Date: Tue, 7 Mar 2023 11:39:16 +0100 Subject: [PATCH] Fix - Fixed handling of default passed params --- CHANGELOG.md | 4 + docs/conf.py | 2 +- setup.py | 2 +- src/emhass/command_line.py | 15 ++-- src/emhass/utils.py | 141 +++++++++++++------------------ tests/test_command_line_utils.py | 23 +++-- 6 files changed, 90 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 355dcf1a..5f80e2c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [0.4.2] - 2023-03-07 +### Fix +- Fixed handling of default passed params. + ## [0.4.1] - 2023-03-06 ### Improvement - Improved the documentation and the in-code docstrings. diff --git a/docs/conf.py b/docs/conf.py index 9131cd37..7e1e08ac 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,7 +22,7 @@ author = 'David HERNANDEZ' # The full version, including alpha/beta/rc tags -release = '0.4.1' +release = '0.4.2' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index e59c6836..9f6b8bf7 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name='emhass', # Required - version='0.4.1', # Required + version='0.4.2', # Required description='An Energy Management System for Home Assistant', # Optional long_description=long_description, # Optional long_description_content_type='text/markdown', # Optional (see note above) diff --git a/src/emhass/command_line.py b/src/emhass/command_line.py index 35f7d237..0d674cd0 100644 --- a/src/emhass/command_line.py +++ b/src/emhass/command_line.py @@ -461,20 +461,21 @@ def publish_data(input_data_dict: dict, logger: logging.Logger, if idx_closest == -1: idx_closest = opt_res_latest.index.get_indexer([now_precise], method='nearest')[0] # Publish PV forecast - custom_pv_forecast_id = input_data_dict['params']['passed_data']['custom_pv_forecast_id'] + params = json.loads(input_data_dict['params']) + custom_pv_forecast_id = params['passed_data']['custom_pv_forecast_id'] input_data_dict['rh'].post_data(opt_res_latest['P_PV'], idx_closest, custom_pv_forecast_id["entity_id"], custom_pv_forecast_id["unit_of_measurement"], custom_pv_forecast_id["friendly_name"]) # Publish Load forecast - custom_load_forecast_id = input_data_dict['params']['passed_data']['custom_load_forecast_id'] + custom_load_forecast_id = params['passed_data']['custom_load_forecast_id'] input_data_dict['rh'].post_data(opt_res_latest['P_Load'], idx_closest, custom_load_forecast_id["entity_id"], custom_load_forecast_id["unit_of_measurement"], custom_load_forecast_id["friendly_name"]) cols_published = ['P_PV', 'P_Load'] # Publish deferrable loads - custom_deferrable_forecast_id = input_data_dict['params']['passed_data']['custom_deferrable_forecast_id'] + custom_deferrable_forecast_id = params['passed_data']['custom_deferrable_forecast_id'] for k in range(input_data_dict['opt'].optim_conf['num_def_loads']): if "P_deferrable{}".format(k) not in opt_res_latest.columns: logger.error("P_deferrable{}".format(k)+" was not found in results DataFrame. Optimization task may need to be relaunched or it did not converged to a solution.") @@ -489,27 +490,27 @@ def publish_data(input_data_dict: dict, logger: logging.Logger, if 'P_batt' not in opt_res_latest.columns: logger.error("P_batt was not found in results DataFrame. Optimization task may need to be relaunched or it did not converged to a solution.") else: - custom_batt_forecast_id = input_data_dict['params']['passed_data']['custom_batt_forecast_id'] + custom_batt_forecast_id = params['passed_data']['custom_batt_forecast_id'] input_data_dict['rh'].post_data(opt_res_latest['P_batt'], idx_closest, custom_batt_forecast_id["entity_id"], custom_batt_forecast_id["unit_of_measurement"], custom_batt_forecast_id["friendly_name"]) cols_published = cols_published+["P_batt"] - custom_batt_soc_forecast_id = input_data_dict['params']['passed_data']['custom_batt_soc_forecast_id'] + custom_batt_soc_forecast_id = params['passed_data']['custom_batt_soc_forecast_id'] input_data_dict['rh'].post_data(opt_res_latest['SOC_opt']*100, idx_closest, custom_batt_soc_forecast_id["entity_id"], custom_batt_soc_forecast_id["unit_of_measurement"], custom_batt_soc_forecast_id["friendly_name"]) cols_published = cols_published+["SOC_opt"] # Publish grid power - custom_grid_forecast_id = input_data_dict['params']['passed_data']['custom_grid_forecast_id'] + custom_grid_forecast_id = params['passed_data']['custom_grid_forecast_id'] input_data_dict['rh'].post_data(opt_res_latest['P_grid'], idx_closest, custom_grid_forecast_id["entity_id"], custom_grid_forecast_id["unit_of_measurement"], custom_grid_forecast_id["friendly_name"]) cols_published = cols_published+["P_grid"] # Publish total value of cost function - custom_cost_fun_id = input_data_dict['params']['passed_data']['custom_cost_fun_id'] + custom_cost_fun_id = params['passed_data']['custom_cost_fun_id'] col_cost_fun = [i for i in opt_res_latest.columns if 'cost_fun_' in i] input_data_dict['rh'].post_data(opt_res_latest[col_cost_fun], idx_closest, custom_cost_fun_id["entity_id"], diff --git a/src/emhass/utils.py b/src/emhass/utils.py index 0273df70..acbc6369 100644 --- a/src/emhass/utils.py +++ b/src/emhass/utils.py @@ -95,7 +95,7 @@ def get_forecast_dates(freq: int, delta_forecast: int, freq=freq).round(freq) return forecast_dates -def treat_runtimeparams(runtimeparams: str, params:str, retrieve_hass_conf: dict, optim_conf: dict, plant_conf: dict, +def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dict, optim_conf: dict, plant_conf: dict, set_type: str, logger: logging.Logger) -> Tuple[str, dict]: """ Treat the passed optimization runtime parameters. @@ -118,13 +118,32 @@ def treat_runtimeparams(runtimeparams: str, params:str, retrieve_hass_conf: dict :rtype: Tuple[str, dict] """ + if (params != None) and (params != 'null'): + params = json.loads(params) + else: + params = {} + # Some default data needed + custom_deferrable_forecast_id = [] + for k in range(optim_conf['num_def_loads']): + custom_deferrable_forecast_id.append({ + "entity_id": "sensor.p_deferrable{}".format(k), + "unit_of_measurement": "W", + "friendly_name": "Deferrable Load {}".format(k) + }) + default_passed_dict = {'custom_pv_forecast_id': {"entity_id": "sensor.p_pv_forecast", "unit_of_measurement": "W", "friendly_name": "PV Power Forecast"}, + 'custom_load_forecast_id': {"entity_id": "sensor.p_load_forecast", "unit_of_measurement": "W", "friendly_name": "Load Power Forecast"}, + 'custom_batt_forecast_id': {"entity_id": "sensor.p_batt_forecast", "unit_of_measurement": "W", "friendly_name": "Battery Power Forecast"}, + 'custom_batt_soc_forecast_id': {"entity_id": "sensor.soc_batt_forecast", "unit_of_measurement": "%", "friendly_name": "Battery SOC Forecast"}, + 'custom_grid_forecast_id': {"entity_id": "sensor.p_grid_forecast", "unit_of_measurement": "W", "friendly_name": "Grid Power Forecast"}, + 'custom_cost_fun_id': {"entity_id": "sensor.total_cost_fun_value", "unit_of_measurement": "", "friendly_name": "Total cost function value"}, + 'custom_deferrable_forecast_id': custom_deferrable_forecast_id} + if 'passed_data' in params.keys(): + for key, value in default_passed_dict.items(): + params['passed_data'][key] = value + else: + params['passed_data'] = default_passed_dict if runtimeparams is not None: runtimeparams = json.loads(runtimeparams) - if (params != None) and (params != 'null'): - params = json.loads(params) - else: - params = {'passed_data':{'pv_power_forecast':None,'load_power_forecast':None,'load_cost_forecast':None,'prod_price_forecast':None, - 'prediction_horizon':None,'soc_init':None,'soc_final':None,'def_total_hours':None,'alpha':None,'beta':None}} freq = int(retrieve_hass_conf['freq'].seconds/60.0) delta_forecast = int(optim_conf['delta_forecast'].days) forecast_dates = get_forecast_dates(freq, delta_forecast) @@ -161,6 +180,13 @@ def treat_runtimeparams(runtimeparams: str, params:str, retrieve_hass_conf: dict beta = runtimeparams['beta'] params['passed_data']['beta'] = beta forecast_dates = copy.deepcopy(forecast_dates)[0:prediction_horizon] + else: + params['passed_data']['prediction_horizon'] = None + params['passed_data']['soc_init'] = None + params['passed_data']['soc_final'] = None + params['passed_data']['def_total_hours'] = None + params['passed_data']['alpha'] = None + params['passed_data']['beta'] = None # Treat passed forecast data lists if 'pv_power_forecast' in runtimeparams.keys(): if type(runtimeparams['pv_power_forecast']) == list and len(runtimeparams['pv_power_forecast']) >= len(forecast_dates): @@ -174,6 +200,8 @@ def treat_runtimeparams(runtimeparams: str, params:str, retrieve_hass_conf: dict logger.warning("There are non numeric values on the passed data for pv_power_forecast, check for missing values (nans, null, etc)") for x in list_non_digits: logger.warning("This value in pv_power_forecast was detected as non digits: "+str(x)) + else: + params['passed_data']['pv_power_forecast'] = None if 'load_power_forecast' in runtimeparams.keys(): if type(runtimeparams['load_power_forecast']) == list and len(runtimeparams['load_power_forecast']) >= len(forecast_dates): params['passed_data']['load_power_forecast'] = runtimeparams['load_power_forecast'] @@ -186,6 +214,8 @@ def treat_runtimeparams(runtimeparams: str, params:str, retrieve_hass_conf: dict logger.warning("There are non numeric values on the passed data for load_power_forecast, check for missing values (nans, null, etc)") for x in list_non_digits: logger.warning("This value in load_power_forecast was detected as non digits: "+str(x)) + else: + params['passed_data']['load_power_forecast'] = None if 'load_cost_forecast' in runtimeparams.keys(): if type(runtimeparams['load_cost_forecast']) == list and len(runtimeparams['load_cost_forecast']) >= len(forecast_dates): params['passed_data']['load_cost_forecast'] = runtimeparams['load_cost_forecast'] @@ -198,6 +228,8 @@ def treat_runtimeparams(runtimeparams: str, params:str, retrieve_hass_conf: dict logger.warning("There are non numeric values on the passed data or load_cost_forecast, check for missing values (nans, null, etc)") for x in list_non_digits: logger.warning("This value in load_cost_forecast was detected as non digits: "+str(x)) + else: + params['passed_data']['load_cost_forecast'] = None if 'prod_price_forecast' in runtimeparams.keys(): if type(runtimeparams['prod_price_forecast']) == list and len(runtimeparams['prod_price_forecast']) >= len(forecast_dates): params['passed_data']['prod_price_forecast'] = runtimeparams['prod_price_forecast'] @@ -210,8 +242,10 @@ def treat_runtimeparams(runtimeparams: str, params:str, retrieve_hass_conf: dict logger.warning("There are non numeric values on the passed data for prod_price_forecast, check for missing values (nans, null, etc)") for x in list_non_digits: logger.warning("This value in prod_price_forecast was detected as non digits: "+str(x)) - # Treat passed data for forecast model fit - if set_type == 'forecast-model-fit': + else: + params['passed_data']['prod_price_forecast'] = None + # Treat passed data for forecast model fit/predict/tune + if set_type == 'forecast-model-fit' or set_type == 'forecast-model-predict' or set_type == 'forecast-model-tune': if 'days_to_retrieve' not in runtimeparams.keys(): days_to_retrieve = 30 else: @@ -247,8 +281,6 @@ def treat_runtimeparams(runtimeparams: str, params:str, retrieve_hass_conf: dict else: perform_backtest = runtimeparams['perform_backtest'] params['passed_data']['perform_backtest'] = perform_backtest - # Treat passed data for forecast model predict - if set_type == 'forecast-model-predict': if 'model_predict_publish' not in runtimeparams.keys(): model_predict_publish = False else: @@ -290,79 +322,22 @@ def treat_runtimeparams(runtimeparams: str, params:str, retrieve_hass_conf: dict retrieve_hass_conf['solar_forecast_kwp'] = runtimeparams['solar_forecast_kwp'] optim_conf['weather_forecast_method'] = 'solar.forecast' # Treat custom entities id's and friendly names for variables - if 'custom_pv_forecast_id' not in runtimeparams.keys(): - custom_pv_forecast_id = { - "entity_id": "sensor.p_pv_forecast", - "unit_of_measurement": "W", - "friendly_name": "PV Power Forecast" - } - else: - custom_pv_forecast_id = runtimeparams['custom_pv_forecast_id'] - params['passed_data']['custom_pv_forecast_id'] = custom_pv_forecast_id - - if 'custom_load_forecast_id' not in runtimeparams.keys(): - custom_load_forecast_id = { - "entity_id": "sensor.p_load_forecast", - "unit_of_measurement": "W", - "friendly_name": "Load Power Forecast" - } - else: - custom_load_forecast_id = runtimeparams['custom_load_forecast_id'] - params['passed_data']['custom_load_forecast_id'] = custom_load_forecast_id - - if 'custom_batt_forecast_id' not in runtimeparams.keys(): - custom_batt_forecast_id = { - "entity_id": "sensor.p_batt_forecast", - "unit_of_measurement": "W", - "friendly_name": "Battery Power Forecast" - } - else: - custom_batt_forecast_id = runtimeparams['custom_batt_forecast_id'] - params['passed_data']['custom_batt_forecast_id'] = custom_batt_forecast_id - - if 'custom_batt_soc_forecast_id' not in runtimeparams.keys(): - custom_batt_soc_forecast_id = { - "entity_id": "sensor.soc_batt_forecast", - "unit_of_measurement": "%", - "friendly_name": "Battery SOC Forecast" - } - else: - custom_batt_soc_forecast_id = runtimeparams['custom_batt_soc_forecast_id'] - params['passed_data']['custom_batt_soc_forecast_id'] = custom_batt_soc_forecast_id - - if 'custom_grid_forecast_id' not in runtimeparams.keys(): - custom_grid_forecast_id = { - "entity_id": "sensor.p_grid_forecast", - "unit_of_measurement": "W", - "friendly_name": "Grid Power Forecast" - } - else: - custom_grid_forecast_id = runtimeparams['custom_grid_forecast_id'] - params['passed_data']['custom_grid_forecast_id'] = custom_grid_forecast_id - - if 'custom_cost_fun_id' not in runtimeparams.keys(): - custom_cost_fun_id = { - "entity_id": "sensor.total_cost_fun_value", - "unit_of_measurement": "", - "friendly_name": "Total cost function value" - } - else: - custom_cost_fun_id = runtimeparams['custom_cost_fun_id'] - params['passed_data']['custom_cost_fun_id'] = custom_cost_fun_id - - if 'custom_deferrable_forecast_id' not in runtimeparams.keys(): - custom_deferrable_forecast_id = [] - for k in range(optim_conf['num_def_loads']): - custom_deferrable_forecast_id.append({ - "entity_id": "sensor.p_deferrable{}".format(k), - "unit_of_measurement": "W", - "friendly_name": "Deferrable Load {}".format(k) - }) - else: - custom_deferrable_forecast_id = runtimeparams['custom_deferrable_forecast_id'] - params['passed_data']['custom_deferrable_forecast_id'] = custom_deferrable_forecast_id - - params = json.dumps(params) + if 'custom_pv_forecast_id' in runtimeparams.keys(): + params['passed_data']['custom_pv_forecast_id'] = runtimeparams['custom_pv_forecast_id'] + if 'custom_load_forecast_id' in runtimeparams.keys(): + params['passed_data']['custom_load_forecast_id'] = runtimeparams['custom_load_forecast_id'] + if 'custom_batt_forecast_id' in runtimeparams.keys(): + params['passed_data']['custom_batt_forecast_id'] = runtimeparams['custom_batt_forecast_id'] + if 'custom_batt_soc_forecast_id' in runtimeparams.keys(): + params['passed_data']['custom_batt_soc_forecast_id'] = runtimeparams['custom_batt_soc_forecast_id'] + if 'custom_grid_forecast_id' in runtimeparams.keys(): + params['passed_data']['custom_grid_forecast_id'] = runtimeparams['custom_grid_forecast_id'] + if 'custom_cost_fun_id' in runtimeparams.keys(): + params['passed_data']['custom_cost_fun_id'] = runtimeparams['custom_cost_fun_id'] + if 'custom_deferrable_forecast_id' in runtimeparams.keys(): + params['passed_data']['custom_deferrable_forecast_id'] = runtimeparams['custom_deferrable_forecast_id'] + # Serialize the final params + params = json.dumps(params) return params, retrieve_hass_conf, optim_conf def get_yaml_parse(config_path: str, use_secrets: Optional[bool] = True, diff --git a/tests/test_command_line_utils.py b/tests/test_command_line_utils.py index e9516969..3fd5d71b 100644 --- a/tests/test_command_line_utils.py +++ b/tests/test_command_line_utils.py @@ -96,7 +96,7 @@ def test_set_input_data_dict(self): self.assertTrue(input_data_dict['df_input_data_dayahead'].index.freq is not None) self.assertTrue(input_data_dict['df_input_data_dayahead'].isnull().sum().sum()==0) self.assertTrue(len(input_data_dict['df_input_data_dayahead'])==10) # The default value for prediction_horizon - action = 'dayahead-optim' + action = 'naive-mpc-optim' runtimeparams['prediction_horizon'] = 10 runtimeparams_json = json.dumps(runtimeparams) params = copy.deepcopy(json.loads(self.params_json)) @@ -209,16 +209,29 @@ def test_naive_mpc_optim(self): params_json = json.dumps(params) input_data_dict = set_input_data_dict(config_path, base_path, costfun, params_json, self.runtimeparams_json, action, logger, get_data_from_file=True) - opt_res = dayahead_forecast_optim(input_data_dict, logger, debug=True) + opt_res = naive_mpc_optim(input_data_dict, logger, debug=True) + action = 'publish-data' + input_data_dict = set_input_data_dict(config_path, base_path, costfun, params_json, None, + action, logger, get_data_from_file=True) opt_res_first = publish_data(input_data_dict, logger, opt_res_latest=opt_res) self.assertTrue(len(opt_res_first)==1) + action = 'naive-mpc-optim' params = copy.deepcopy(json.loads(self.params_json)) params['retrieve_hass_conf'][8]['method_ts_round'] = 'last' params['optim_conf'][0]['set_use_battery'] = True params_json = json.dumps(params) input_data_dict = set_input_data_dict(config_path, base_path, costfun, params_json, self.runtimeparams_json, action, logger, get_data_from_file=True) - opt_res = dayahead_forecast_optim(input_data_dict, logger, debug=True) + opt_res = naive_mpc_optim(input_data_dict, logger, debug=True) + action = 'publish-data' + input_data_dict = set_input_data_dict(config_path, base_path, costfun, params_json, None, + action, logger, get_data_from_file=True) + opt_res_last = publish_data(input_data_dict, logger, opt_res_latest=opt_res) + self.assertTrue(len(opt_res_last)==1) + # Reproduce when trying to publish data but params=None and runtimeparams=None + action = 'publish-data' + input_data_dict = set_input_data_dict(config_path, base_path, costfun, None, None, + action, logger, get_data_from_file=True) opt_res_last = publish_data(input_data_dict, logger, opt_res_latest=opt_res) self.assertTrue(len(opt_res_last)==1) @@ -227,7 +240,7 @@ def test_forecast_model_fit_predict_tune(self): base_path = str(config_path.parent) costfun = 'profit' action = 'forecast-model-fit' # fit, predict and tune methods - params = copy.deepcopy(json.loads(self.params_json)) + params = TestCommandLineUtils.get_test_params() runtimeparams = { "days_to_retrieve": 20, "model_type": "load_forecast", @@ -242,7 +255,7 @@ def test_forecast_model_fit_predict_tune(self): "model_predict_friendly_name": "Load Power Forecast KNN regressor" } runtimeparams_json = json.dumps(runtimeparams) - params['passed_data'] = runtimeparams + # params['passed_data'] = runtimeparams params['optim_conf'][8]['load_forecast_method'] = 'skforecast' params_json = json.dumps(params) input_data_dict = set_input_data_dict(config_path, base_path, costfun, params_json, runtimeparams_json,