From 33a6d6f6a3a4f41f2b53c25c710c29ee0841a432 Mon Sep 17 00:00:00 2001 From: Pedro Venda Date: Fri, 24 May 2024 16:26:48 +0100 Subject: [PATCH 1/3] feature: interpret load sensors in kW rather than W --- config_emhass.yaml | 1 + docs/config.md | 1 + docs/differences.md | 1 + options.json | 1 + scripts/load_clustering.py | 4 ++-- scripts/load_forecast_sklearn.py | 6 +++--- scripts/optim_results_analysis.py | 4 ++-- scripts/special_config_analysis.py | 4 ++-- scripts/use_cases_analysis.py | 4 ++-- src/emhass/command_line.py | 6 +++--- src/emhass/forecast.py | 2 +- src/emhass/retrieve_hass.py | 30 ++++++++++++++++++++++-------- src/emhass/utils.py | 3 +++ tests/test_forecast.py | 6 +++--- tests/test_optimization.py | 2 +- tests/test_retrieve_hass.py | 9 +++++---- 16 files changed, 53 insertions(+), 31 deletions(-) diff --git a/config_emhass.yaml b/config_emhass.yaml index b73e6d99..e6e8c7f3 100644 --- a/config_emhass.yaml +++ b/config_emhass.yaml @@ -5,6 +5,7 @@ retrieve_hass_conf: days_to_retrieve: 2 # We will retrieve data from now and up to days_to_retrieve days var_PV: 'sensor.power_photovoltaics' # Photovoltaic produced power sensor in Watts var_load: 'sensor.power_load_no_var_loads' # Household power consumption sensor in Watts (deferrable loads should be substracted) + load_sensor_kw: False # photovoltaic and load_no_var_loads sensors in kW rather than W load_negative: False # Set to True if the retrived load variable is negative by convention set_zero_min: True # A special treatment for a minimum value saturation to zero. Values below zero are replaced by nans var_replace_zero: # A list of retrived variables that we would want to replace nans with zeros diff --git a/docs/config.md b/docs/config.md index ef46ad20..9dfa2282 100644 --- a/docs/config.md +++ b/docs/config.md @@ -16,6 +16,7 @@ These are the parameters that we will need to define to retrieve data from Home - `days_to_retrieve`: We will retrieve data from now and up to days_to_retrieve days. Defaults to 2. - `var_PV`: This is the name of the photovoltaic produced power sensor in Watts from Home Assistant. For example: 'sensor.power_photovoltaics'. - `var_load`: The name of the household power consumption sensor in Watts from Home Assistant. The deferrable loads that we will want to include in the optimization problem should be substracted from this sensor in HASS. For example: 'sensor.power_load_no_var_loads' +- `load_sensor_kw`: Set this parameter to True if the load and PV sensors are in kW rather than W. Defaults to False. - `load_negative`: Set this parameter to True if the retrived load variable is negative by convention. Defaults to False. - `set_zero_min`: Set this parameter to True to give a special treatment for a minimum value saturation to zero for power consumption data. Values below zero are replaced by nans. Defaults to True. - `var_replace_zero`: The list of retrieved variables that we would want to replace nans (if they exist) with zeros. For example: diff --git a/docs/differences.md b/docs/differences.md index 68fbf5f1..2e1c8dc6 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -20,6 +20,7 @@ See bellow for a list of associations between the parameters from `config_emhass | retrieve_hass_conf | var_PV | sensor_power_photovoltaics | | | retrieve_hass_conf | var_load | sensor_power_load_no_var_loads | | | retrieve_hass_conf | load_negative | load_negative | | +| retrieve_hass_conf | load_sensor_kw | load_sensor_kw | | | retrieve_hass_conf | set_zero_min | set_zero_min | | | retrieve_hass_conf | method_ts_round | method_ts_round | | | params_secrets | solcast_api_key | optional_solcast_api_key | | diff --git a/options.json b/options.json index b0b55076..4bf9e813 100644 --- a/options.json +++ b/options.json @@ -22,6 +22,7 @@ "weight_battery_charge": 1.0, "sensor_power_photovoltaics": "sensor.power_photovoltaics", "sensor_power_load_no_var_loads": "sensor.power_load_no_var_loads", + "load_sensor_kw": false, "load_negative": false, "set_zero_min": true, "number_of_deferrable_loads": 2, diff --git a/scripts/load_clustering.py b/scripts/load_clustering.py index db92ea2e..72c9493c 100644 --- a/scripts/load_clustering.py +++ b/scripts/load_clustering.py @@ -65,7 +65,7 @@ days_list = get_days_list(days_to_retrieve) var_list = [var_model] - rh.get_data(days_list, var_list) + rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']) with open(data_path, 'wb') as fid: pickle.dump((rh.df_final, var_model), fid, pickle.HIGHEST_PROTOCOL) @@ -144,4 +144,4 @@ # data_lag['cluster_group_tslearn_sdtw'] = y_pred # fig = px.scatter(data_lag, x='power_load y(t)', y='power_load y(t+1)', color='cluster_group_tslearn_sdtw', template=template) - # fig.show() \ No newline at end of file + # fig.show() diff --git a/scripts/load_forecast_sklearn.py b/scripts/load_forecast_sklearn.py index 82f5b3b3..2ea131d6 100644 --- a/scripts/load_forecast_sklearn.py +++ b/scripts/load_forecast_sklearn.py @@ -74,7 +74,7 @@ def neg_r2_score(y_true, y_pred): days_list = get_days_list(days_to_retrieve) var_list = [var_model] - rh.get_data(days_list, var_list) + rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']) with open(data_path, 'wb') as fid: pickle.dump((rh.df_final, var_model), fid, pickle.HIGHEST_PROTOCOL) @@ -257,7 +257,7 @@ def neg_r2_score(y_true, y_pred): days_list = get_days_list(days_needed) var_model = retrieve_hass_conf['var_load'] var_list = [var_model] - rh.get_data(days_list, var_list) + rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']) data_last_window = copy.deepcopy(rh.df_final) data_last_window = add_date_features(data_last_window) @@ -275,4 +275,4 @@ def neg_r2_score(y_true, y_pred): fig.update_yaxes(title_text = "Power (W)") fig.update_xaxes(title_text = "Time") fig.show() - fig.write_image(emhass_conf['root_path'] / "docs/images/load_forecast_production.svg", width=1080, height=0.8*1080) \ No newline at end of file + fig.write_image(emhass_conf['root_path'] / "docs/images/load_forecast_production.svg", width=1080, height=0.8*1080) diff --git a/scripts/optim_results_analysis.py b/scripts/optim_results_analysis.py index 173c7045..77d5e66b 100644 --- a/scripts/optim_results_analysis.py +++ b/scripts/optim_results_analysis.py @@ -64,7 +64,7 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, days_list = get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False) + minimal_response=False, significant_changes_only=False,load_sensor_kw=retrieve_hass_conf['load_sensor_kw']) rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], set_zero_min = retrieve_hass_conf['set_zero_min'], var_replace_zero = retrieve_hass_conf['var_replace_zero'], @@ -130,4 +130,4 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, print(opt_res_dah) if save_html: - opt_res_dah.to_html('opt_res_dah.html') \ No newline at end of file + opt_res_dah.to_html('opt_res_dah.html') diff --git a/scripts/special_config_analysis.py b/scripts/special_config_analysis.py index ace61ab8..2a10155c 100644 --- a/scripts/special_config_analysis.py +++ b/scripts/special_config_analysis.py @@ -110,7 +110,7 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, days_list = get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False) + minimal_response=False, significant_changes_only=False,load_sensor_kw=retrieve_hass_conf['load_sensor_kw']) rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], set_zero_min = retrieve_hass_conf['set_zero_min'], var_replace_zero = retrieve_hass_conf['var_replace_zero'], @@ -185,4 +185,4 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, fig_res_mpc.layout.template = template fig_res_mpc.update_yaxes(title_text = "Powers (W)") fig_res_mpc.update_xaxes(title_text = "Time") - fig_res_mpc.show() \ No newline at end of file + fig_res_mpc.show() diff --git a/scripts/use_cases_analysis.py b/scripts/use_cases_analysis.py index 0030784e..ab77c243 100644 --- a/scripts/use_cases_analysis.py +++ b/scripts/use_cases_analysis.py @@ -56,7 +56,7 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, days_list = get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False) + minimal_response=False, significant_changes_only=False,load_sensor_kw=retrieve_hass_conf['load_sensor_kw']) rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], set_zero_min = retrieve_hass_conf['set_zero_min'], var_replace_zero = retrieve_hass_conf['var_replace_zero'], @@ -177,4 +177,4 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, width=1080, height=0.8*1080) print("System with: PV, Battery, two deferrable loads, dayahead optimization, profit >> total cost function sum: "+\ - str(opt_res_dah['cost_profit'].sum())) \ No newline at end of file + str(opt_res_dah['cost_profit'].sum())) diff --git a/src/emhass/command_line.py b/src/emhass/command_line.py index 7bb455b3..89ab0690 100644 --- a/src/emhass/command_line.py +++ b/src/emhass/command_line.py @@ -81,7 +81,7 @@ def set_input_data_dict(emhass_conf: dict, costfun: str, retrieve_hass_conf["days_to_retrieve"]) var_list = [retrieve_hass_conf["var_load"], retrieve_hass_conf["var_PV"]] - if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False): + if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']): return False if not rh.prepare_data(retrieve_hass_conf["var_load"], load_negative=retrieve_hass_conf["load_negative"], @@ -129,7 +129,7 @@ def set_input_data_dict(emhass_conf: dict, costfun: str, days_list = utils.get_days_list(1) var_list = [retrieve_hass_conf["var_load"], retrieve_hass_conf["var_PV"]] - if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False): + if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']): return False if not rh.prepare_data(retrieve_hass_conf["var_load"], load_negative=retrieve_hass_conf["load_negative"], @@ -175,7 +175,7 @@ def set_input_data_dict(emhass_conf: dict, costfun: str, else: days_list = utils.get_days_list(days_to_retrieve) var_list = [var_model] - if not rh.get_data(days_list, var_list): + if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']): return False df_input_data = rh.df_final.copy() elif set_type == "regressor-model-fit" or set_type == "regressor-model-predict": diff --git a/src/emhass/forecast.py b/src/emhass/forecast.py index c0c74a85..5b493b07 100644 --- a/src/emhass/forecast.py +++ b/src/emhass/forecast.py @@ -643,7 +643,7 @@ def get_load_forecast(self, days_min_load_forecast: Optional[int] = 3, method: O self.var_load_new = self.var_load+'_positive' else: days_list = get_days_list(days_min_load_forecast) - if not rh.get_data(days_list, var_list): + if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = self.retrieve_hass_conf['load_sensor_kw']): return False if not rh.prepare_data( self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'], diff --git a/src/emhass/retrieve_hass.py b/src/emhass/retrieve_hass.py index 476a7c9b..bb6fad80 100644 --- a/src/emhass/retrieve_hass.py +++ b/src/emhass/retrieve_hass.py @@ -67,7 +67,7 @@ def __init__(self, hass_url: str, long_lived_token: str, freq: pd.Timedelta, def get_data(self, days_list: pd.date_range, var_list: list, minimal_response: Optional[bool] = False, significant_changes_only: Optional[bool] = False, - test_url: Optional[str] = "empty") -> None: + test_url: Optional[str] = "empty", load_sensor_kw: Optional[bool] = False) -> None: r""" Retrieve the actual data from hass. @@ -84,6 +84,9 @@ def get_data(self, days_list: pd.date_range, var_list: list, :param significant_changes_only: Retrieve significant changes only \ using the hass restful API, defaults to False :type significant_changes_only: bool, optional + :param load_sensor_kw: Set to True to interpret the load variable in \ + kW rather than the default W, defaults to False + :type load_sensor_kw: bool, optional :return: The DataFrame populated with the retrieved data from hass :rtype: pandas.DataFrame @@ -181,13 +184,24 @@ def get_data(self, days_list: pd.date_range, var_list: list, ts = pd.to_datetime(pd.date_range(start=from_date, end=to_date, freq=self.freq), format='%Y-%d-%m %H:%M').round(self.freq, ambiguous='infer', nonexistent='shift_forward') df_day = pd.DataFrame(index = ts) - # Caution with undefined string data: unknown, unavailable, etc. - df_tp = ( - df_raw.copy()[["state"]] - .replace(["unknown", "unavailable", ""], np.nan) - .astype(float) - .rename(columns={"state": var}) - ) + # if load sensors are in kW, multiply the state entry by 1000 + if load_sensor_kw: + # Caution with undefined string data: unknown, unavailable, etc. + df_tp = ( + df_raw.copy()[["state"]] + .replace(["unknown", "unavailable", ""], np.nan) + .astype(float) + .rename(columns={"state": var}) + .mul(1000) + ) + else: + # Caution with undefined string data: unknown, unavailable, etc. + df_tp = ( + df_raw.copy()[["state"]] + .replace(["unknown", "unavailable", ""], np.nan) + .astype(float) + .rename(columns={"state": var}) + ) # Setting index, resampling and concatenation df_tp.set_index( pd.to_datetime(df_raw["last_changed"], format="ISO8601"), diff --git a/src/emhass/utils.py b/src/emhass/utils.py index f32a136d..2d533f5b 100644 --- a/src/emhass/utils.py +++ b/src/emhass/utils.py @@ -743,6 +743,9 @@ def build_params(params: dict, params_secrets: dict, options: dict, addon: int, params["retrieve_hass_conf"]["var_load"] = options.get( "sensor_power_load_no_var_loads", params["retrieve_hass_conf"]["var_load"] ) + params["retrieve_hass_conf"]["load_sensor_kw"] = options.get( + "load_sensor_kw", params["retrieve_hass_conf"]["load_sensor_kw"] + ) params["retrieve_hass_conf"]["load_negative"] = options.get( "load_negative", params["retrieve_hass_conf"]["load_negative"] ) diff --git a/tests/test_forecast.py b/tests/test_forecast.py index 2c4b6414..946e33b4 100644 --- a/tests/test_forecast.py +++ b/tests/test_forecast.py @@ -63,7 +63,7 @@ def setUp(self): self.days_list = utils.get_days_list(self.retrieve_hass_conf['days_to_retrieve']) self.var_list = [self.retrieve_hass_conf['var_load'], self.retrieve_hass_conf['var_PV']] self.rh.get_data(self.days_list, self.var_list, - minimal_response=False, significant_changes_only=False) + minimal_response=False, significant_changes_only=False, load_sensor_kw = self.retrieve_hass_conf['load_sensor_kw']) self.rh.prepare_data(self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'], set_zero_min = self.retrieve_hass_conf['set_zero_min'], var_replace_zero = self.retrieve_hass_conf['var_replace_zero'], @@ -221,7 +221,7 @@ def test_get_forecasts_with_lists(self): days_list = utils.get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False) + minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']) rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], set_zero_min = retrieve_hass_conf['set_zero_min'], var_replace_zero = retrieve_hass_conf['var_replace_zero'], @@ -358,7 +358,7 @@ def test_get_forecasts_with_lists_special_case(self): days_list = utils.get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False) + minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']) rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], set_zero_min = retrieve_hass_conf['set_zero_min'], var_replace_zero = retrieve_hass_conf['var_replace_zero'], diff --git a/tests/test_optimization.py b/tests/test_optimization.py index 0cf77e46..e2f24e8a 100644 --- a/tests/test_optimization.py +++ b/tests/test_optimization.py @@ -45,7 +45,7 @@ def setUp(self): self.days_list = get_days_list(self.retrieve_hass_conf['days_to_retrieve']) self.var_list = [self.retrieve_hass_conf['var_load'], self.retrieve_hass_conf['var_PV']] self.rh.get_data(self.days_list, self.var_list, - minimal_response=False, significant_changes_only=False) + minimal_response=False, significant_changes_only=False,load_sensor_kw=self.retrieve_hass_conf['load_sensor_kw']) self.rh.prepare_data(self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'], set_zero_min = self.retrieve_hass_conf['set_zero_min'], var_replace_zero = self.retrieve_hass_conf['var_replace_zero'], diff --git a/tests/test_retrieve_hass.py b/tests/test_retrieve_hass.py index dd082909..0fc3c1d3 100644 --- a/tests/test_retrieve_hass.py +++ b/tests/test_retrieve_hass.py @@ -47,7 +47,7 @@ def setUp(self): self.days_list = get_days_list(self.retrieve_hass_conf['days_to_retrieve']) self.var_list = [self.retrieve_hass_conf['var_load'], self.retrieve_hass_conf['var_PV']] self.rh.get_data(self.days_list, self.var_list, - minimal_response=False, significant_changes_only=False) + minimal_response=False, significant_changes_only=False, load_sensor_kw=self.retrieve_hass_conf['load_sensor_kw']) if save_data_to_file: with open(emhass_conf['data_path'] / 'test_df_final.pkl', 'wb') as outp: pickle.dump((self.rh.df_final, self.days_list, self.var_list), @@ -95,7 +95,7 @@ def test_yaml_parse_wab_server(self): def test_get_data_failed(self): days_list = get_days_list(1) var_list = [self.retrieve_hass_conf['var_load']] - response = self.rh.get_data(days_list, var_list) + response = self.rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw=self.retrieve_hass_conf['load_sensor_kw']) self.assertFalse(response) def test_get_data_mock(self): @@ -107,7 +107,8 @@ def test_get_data_mock(self): m.get(self.retrieve_hass_conf['hass_url'], json=data.json()) self.rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, - test_url=self.retrieve_hass_conf['hass_url']) + test_url=self.retrieve_hass_conf['hass_url'] + load_sensor_kw=self.retrieve_hass_conf['load_sensor_kw']) self.assertIsInstance(self.rh.df_final, type(pd.DataFrame())) self.assertIsInstance(self.rh.df_final.index, pd.core.indexes.datetimes.DatetimeIndex) self.assertIsInstance(self.rh.df_final.index.dtype, pd.core.dtypes.dtypes.DatetimeTZDtype) @@ -182,4 +183,4 @@ def test_publish_data(self): if __name__ == '__main__': unittest.main() ch.close() - logger.removeHandler(ch) \ No newline at end of file + logger.removeHandler(ch) From 1a76fe8bc35199efcb360448a0b82c67a2d0f27c Mon Sep 17 00:00:00 2001 From: Pedro Venda Date: Tue, 28 May 2024 11:24:05 +0100 Subject: [PATCH 2/3] refactored *_in_kw flags to refer to var_PV and var_load indepdendently. modified methods that take runtime parameters to accept a var_model_in_kw boolean for the same purpose. --- config_emhass.yaml | 3 +- docs/config.md | 3 +- docs/differences.md | 3 +- docs/mlforecaster.md | 2 ++ options.json | 3 +- scripts/load_clustering.py | 4 ++- scripts/load_forecast_sklearn.py | 7 +++-- scripts/optim_results_analysis.py | 2 +- scripts/special_config_analysis.py | 2 +- scripts/use_cases_analysis.py | 3 +- src/emhass/command_line.py | 13 ++++++-- src/emhass/forecast.py | 3 +- src/emhass/retrieve_hass.py | 38 ++++++++++++++--------- src/emhass/utils.py | 15 +++++++-- tests/test_command_line_utils.py | 4 +++ tests/test_forecast.py | 11 +++++-- tests/test_machine_learning_forecaster.py | 2 ++ tests/test_optimization.py | 3 +- tests/test_retrieve_hass.py | 24 +++++++++++--- 19 files changed, 106 insertions(+), 39 deletions(-) diff --git a/config_emhass.yaml b/config_emhass.yaml index e6e8c7f3..9c7e3c8d 100644 --- a/config_emhass.yaml +++ b/config_emhass.yaml @@ -4,8 +4,9 @@ retrieve_hass_conf: freq: 30 # The time step to resample retrieved data from hass in minutes days_to_retrieve: 2 # We will retrieve data from now and up to days_to_retrieve days var_PV: 'sensor.power_photovoltaics' # Photovoltaic produced power sensor in Watts + var_PV_in_kw: False # Photovoltaic sensor in kW rather than W var_load: 'sensor.power_load_no_var_loads' # Household power consumption sensor in Watts (deferrable loads should be substracted) - load_sensor_kw: False # photovoltaic and load_no_var_loads sensors in kW rather than W + var_load_in_kw: False # load_no_var_loads in kW rather than W load_negative: False # Set to True if the retrived load variable is negative by convention set_zero_min: True # A special treatment for a minimum value saturation to zero. Values below zero are replaced by nans var_replace_zero: # A list of retrived variables that we would want to replace nans with zeros diff --git a/docs/config.md b/docs/config.md index 9dfa2282..6bbaf131 100644 --- a/docs/config.md +++ b/docs/config.md @@ -15,8 +15,9 @@ These are the parameters that we will need to define to retrieve data from Home - `freq`: The time step to resample retrieved data from hass. This parameter is given in minutes. It should not be defined too low or you will run into memory problems when defining the Linear Programming optimization. Defaults to 30. - `days_to_retrieve`: We will retrieve data from now and up to days_to_retrieve days. Defaults to 2. - `var_PV`: This is the name of the photovoltaic produced power sensor in Watts from Home Assistant. For example: 'sensor.power_photovoltaics'. +- `var_PV_in_kw`: Set this parameter to True if the PV sensor is in kW rather than W. Defaults to False. - `var_load`: The name of the household power consumption sensor in Watts from Home Assistant. The deferrable loads that we will want to include in the optimization problem should be substracted from this sensor in HASS. For example: 'sensor.power_load_no_var_loads' -- `load_sensor_kw`: Set this parameter to True if the load and PV sensors are in kW rather than W. Defaults to False. +- `var_load_in_kw`: Set this parameter to True if the load sensor is in kW rather than W. Defaults to False. - `load_negative`: Set this parameter to True if the retrived load variable is negative by convention. Defaults to False. - `set_zero_min`: Set this parameter to True to give a special treatment for a minimum value saturation to zero for power consumption data. Values below zero are replaced by nans. Defaults to True. - `var_replace_zero`: The list of retrieved variables that we would want to replace nans (if they exist) with zeros. For example: diff --git a/docs/differences.md b/docs/differences.md index 2e1c8dc6..88d76227 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -20,7 +20,8 @@ See bellow for a list of associations between the parameters from `config_emhass | retrieve_hass_conf | var_PV | sensor_power_photovoltaics | | | retrieve_hass_conf | var_load | sensor_power_load_no_var_loads | | | retrieve_hass_conf | load_negative | load_negative | | -| retrieve_hass_conf | load_sensor_kw | load_sensor_kw | | +| retrieve_hass_conf | var_PV_in_kw | var_PV_in_kw | | +| retrieve_hass_conf | var_load_in_kw | var_load_in_kw | | | retrieve_hass_conf | set_zero_min | set_zero_min | | | retrieve_hass_conf | method_ts_round | method_ts_round | | | params_secrets | solcast_api_key | optional_solcast_api_key | | diff --git a/docs/mlforecaster.md b/docs/mlforecaster.md index 1de5d38c..bf32e178 100644 --- a/docs/mlforecaster.md +++ b/docs/mlforecaster.md @@ -26,6 +26,8 @@ The minimum number of `days_to_retrieve` is hard coded to 9 by default. But it i - `var_model`: the name of the sensor to retrieve data from Home Assistant. Example: `sensor.power_load_no_var_loads`. +- `var_model_in_kw`: whether the sensor is in kW. Example: `False`. + - `sklearn_model`: the `scikit-learn` model that will be used. For now only this options are possible: `LinearRegression`, `ElasticNet` and `KNeighborsRegressor`. - `num_lags`: the number of auto-regression lags to consider. A good starting point is to fix this as one day. For example if your time step is 30 minutes, then fix this to 48, if the time step is 1 hour the fix this to 24 and so on. diff --git a/options.json b/options.json index 4bf9e813..e56e0bd7 100644 --- a/options.json +++ b/options.json @@ -22,7 +22,8 @@ "weight_battery_charge": 1.0, "sensor_power_photovoltaics": "sensor.power_photovoltaics", "sensor_power_load_no_var_loads": "sensor.power_load_no_var_loads", - "load_sensor_kw": false, + "var_PV_in_kw": false, + "var_load_in_kw": false, "load_negative": false, "set_zero_min": true, "number_of_deferrable_loads": 2, diff --git a/scripts/load_clustering.py b/scripts/load_clustering.py index 72c9493c..c8af90ff 100644 --- a/scripts/load_clustering.py +++ b/scripts/load_clustering.py @@ -47,6 +47,7 @@ days_to_retrieve = 240 model_type = "load_clustering" var_model = "sensor.power_load_positive" + var_model_in_kw = False data_path = emhass_conf['data_path'] / str('data_train_'+model_type+'.pkl') params = None @@ -65,7 +66,8 @@ days_list = get_days_list(days_to_retrieve) var_list = [var_model] - rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']) + sensors_in_kw = [var_model_in_kw] + rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw) with open(data_path, 'wb') as fid: pickle.dump((rh.df_final, var_model), fid, pickle.HIGHEST_PROTOCOL) diff --git a/scripts/load_forecast_sklearn.py b/scripts/load_forecast_sklearn.py index 2ea131d6..4bcaabda 100644 --- a/scripts/load_forecast_sklearn.py +++ b/scripts/load_forecast_sklearn.py @@ -54,6 +54,7 @@ def neg_r2_score(y_true, y_pred): days_to_retrieve = 240 model_type = "load_forecast" var_model = "sensor.power_load_no_var_loads" + var_model_in_kw = False sklearn_model = "KNeighborsRegressor" num_lags = 48 @@ -74,7 +75,8 @@ def neg_r2_score(y_true, y_pred): days_list = get_days_list(days_to_retrieve) var_list = [var_model] - rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']) + sensors_in_kw = [var_model_in_kw] + rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw) with open(data_path, 'wb') as fid: pickle.dump((rh.df_final, var_model), fid, pickle.HIGHEST_PROTOCOL) @@ -257,7 +259,8 @@ def neg_r2_score(y_true, y_pred): days_list = get_days_list(days_needed) var_model = retrieve_hass_conf['var_load'] var_list = [var_model] - rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']) + sensor_in_kw_list = [retrieve_hass_conf['var_load_in_kw']] + rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list) data_last_window = copy.deepcopy(rh.df_final) data_last_window = add_date_features(data_last_window) diff --git a/scripts/optim_results_analysis.py b/scripts/optim_results_analysis.py index 77d5e66b..8da7d5ae 100644 --- a/scripts/optim_results_analysis.py +++ b/scripts/optim_results_analysis.py @@ -64,7 +64,7 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, days_list = get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False,load_sensor_kw=retrieve_hass_conf['load_sensor_kw']) + minimal_response=False, significant_changes_only=False) rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], set_zero_min = retrieve_hass_conf['set_zero_min'], var_replace_zero = retrieve_hass_conf['var_replace_zero'], diff --git a/scripts/special_config_analysis.py b/scripts/special_config_analysis.py index 2a10155c..497d0419 100644 --- a/scripts/special_config_analysis.py +++ b/scripts/special_config_analysis.py @@ -110,7 +110,7 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, days_list = get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False,load_sensor_kw=retrieve_hass_conf['load_sensor_kw']) + minimal_response=False, significant_changes_only=False) rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], set_zero_min = retrieve_hass_conf['set_zero_min'], var_replace_zero = retrieve_hass_conf['var_replace_zero'], diff --git a/scripts/use_cases_analysis.py b/scripts/use_cases_analysis.py index ab77c243..b5f6d620 100644 --- a/scripts/use_cases_analysis.py +++ b/scripts/use_cases_analysis.py @@ -55,8 +55,9 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, params, emhass_conf, logger) days_list = get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] + sensors_in_kw = [retrieve_hass_conf['var_load_in_kw'], retrieve_hass_conf['var_PV_in_kw']] rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False,load_sensor_kw=retrieve_hass_conf['load_sensor_kw']) + minimal_response=False, significant_changes_only=False,sensor_in_kw_list=sensors_in_kw) rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], set_zero_min = retrieve_hass_conf['set_zero_min'], var_replace_zero = retrieve_hass_conf['var_replace_zero'], diff --git a/src/emhass/command_line.py b/src/emhass/command_line.py index 89ab0690..f5153bfa 100644 --- a/src/emhass/command_line.py +++ b/src/emhass/command_line.py @@ -81,7 +81,9 @@ def set_input_data_dict(emhass_conf: dict, costfun: str, retrieve_hass_conf["days_to_retrieve"]) var_list = [retrieve_hass_conf["var_load"], retrieve_hass_conf["var_PV"]] - if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']): + sensors_in_kw = [retrieve_hass_conf["var_load_in_kw"], + retrieve_hass_conf["var_PV_in_kw"]] + if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw): return False if not rh.prepare_data(retrieve_hass_conf["var_load"], load_negative=retrieve_hass_conf["load_negative"], @@ -129,7 +131,9 @@ def set_input_data_dict(emhass_conf: dict, costfun: str, days_list = utils.get_days_list(1) var_list = [retrieve_hass_conf["var_load"], retrieve_hass_conf["var_PV"]] - if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']): + sensors_in_kw = [retrieve_hass_conf["var_load_in_kw"], + retrieve_hass_conf["var_PV_in_kw"]] + if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw): return False if not rh.prepare_data(retrieve_hass_conf["var_load"], load_negative=retrieve_hass_conf["load_negative"], @@ -165,6 +169,7 @@ def set_input_data_dict(emhass_conf: dict, costfun: str, days_to_retrieve = params["passed_data"]["days_to_retrieve"] model_type = params["passed_data"]["model_type"] var_model = params["passed_data"]["var_model"] + var_model_in_kw = params["passed_data"]["var_model_in_kw"] if get_data_from_file: days_list = None filename = 'data_train_'+model_type+'.pkl' @@ -175,7 +180,8 @@ def set_input_data_dict(emhass_conf: dict, costfun: str, else: days_list = utils.get_days_list(days_to_retrieve) var_list = [var_model] - if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']): + sensors_in_kw = [var_model_in_kw] + if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw): return False df_input_data = rh.df_final.copy() elif set_type == "regressor-model-fit" or set_type == "regressor-model-predict": @@ -404,6 +410,7 @@ def forecast_model_fit(input_data_dict: dict, logger: logging.Logger, data = copy.deepcopy(input_data_dict['df_input_data']) model_type = input_data_dict['params']['passed_data']['model_type'] var_model = input_data_dict['params']['passed_data']['var_model'] + var_model_in_kw = input_data_dict['params']['passed_data']['var_model_in_kw'] sklearn_model = input_data_dict['params']['passed_data']['sklearn_model'] num_lags = input_data_dict['params']['passed_data']['num_lags'] split_date_delta = input_data_dict['params']['passed_data']['split_date_delta'] diff --git a/src/emhass/forecast.py b/src/emhass/forecast.py index 5b493b07..03edd8de 100644 --- a/src/emhass/forecast.py +++ b/src/emhass/forecast.py @@ -643,7 +643,8 @@ def get_load_forecast(self, days_min_load_forecast: Optional[int] = 3, method: O self.var_load_new = self.var_load+'_positive' else: days_list = get_days_list(days_min_load_forecast) - if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw = self.retrieve_hass_conf['load_sensor_kw']): + sensors_in_kw = [self.retrieve_hass_conf['var_load_in_kw']] + if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw): return False if not rh.prepare_data( self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'], diff --git a/src/emhass/retrieve_hass.py b/src/emhass/retrieve_hass.py index bb6fad80..9511a8ff 100644 --- a/src/emhass/retrieve_hass.py +++ b/src/emhass/retrieve_hass.py @@ -67,7 +67,7 @@ def __init__(self, hass_url: str, long_lived_token: str, freq: pd.Timedelta, def get_data(self, days_list: pd.date_range, var_list: list, minimal_response: Optional[bool] = False, significant_changes_only: Optional[bool] = False, - test_url: Optional[str] = "empty", load_sensor_kw: Optional[bool] = False) -> None: + test_url: Optional[str] = "empty", sensor_in_kw_list: Optional[list] = []) -> None: r""" Retrieve the actual data from hass. @@ -84,9 +84,9 @@ def get_data(self, days_list: pd.date_range, var_list: list, :param significant_changes_only: Retrieve significant changes only \ using the hass restful API, defaults to False :type significant_changes_only: bool, optional - :param load_sensor_kw: Set to True to interpret the load variable in \ - kW rather than the default W, defaults to False - :type load_sensor_kw: bool, optional + :param sensor_in_kw_list: Array of flags that determine whether the \ + var_list elements are in kW or W. False if undefined + :type sensor_in_kw_list: list, optional :return: The DataFrame populated with the retrieved data from hass :rtype: pandas.DataFrame @@ -181,19 +181,28 @@ def get_data(self, days_list: pd.date_range, var_list: list, if i == 0: # Defining the DataFrame container from_date = pd.to_datetime(df_raw['last_changed'], format="ISO8601").min() to_date = pd.to_datetime(df_raw['last_changed'], format="ISO8601").max() - ts = pd.to_datetime(pd.date_range(start=from_date, end=to_date, freq=self.freq), + ts = pd.to_datetime(pd.date_range(start=from_date, end=to_date, freq=self.freq), format='%Y-%d-%m %H:%M').round(self.freq, ambiguous='infer', nonexistent='shift_forward') df_day = pd.DataFrame(index = ts) # if load sensors are in kW, multiply the state entry by 1000 - if load_sensor_kw: - # Caution with undefined string data: unknown, unavailable, etc. - df_tp = ( - df_raw.copy()[["state"]] - .replace(["unknown", "unavailable", ""], np.nan) - .astype(float) - .rename(columns={"state": var}) - .mul(1000) - ) + if sensor_in_kw_list: + if sensor_in_kw_list[i]: + # Caution with undefined string data: unknown, unavailable, etc. + df_tp = ( + df_raw.copy()[["state"]] + .replace(["unknown", "unavailable", ""], np.nan) + .astype(float) + .rename(columns={"state": var}) + .mul(1000) + ) + else: + # Caution with undefined string data: unknown, unavailable, etc. + df_tp = ( + df_raw.copy()[["state"]] + .replace(["unknown", "unavailable", ""], np.nan) + .astype(float) + .rename(columns={"state": var}) + ) else: # Caution with undefined string data: unknown, unavailable, etc. df_tp = ( @@ -202,6 +211,7 @@ def get_data(self, days_list: pd.date_range, var_list: list, .astype(float) .rename(columns={"state": var}) ) + # Setting index, resampling and concatenation df_tp.set_index( pd.to_datetime(df_raw["last_changed"], format="ISO8601"), diff --git a/src/emhass/utils.py b/src/emhass/utils.py index 2d533f5b..92cef398 100644 --- a/src/emhass/utils.py +++ b/src/emhass/utils.py @@ -325,11 +325,17 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic else: model_type = runtimeparams["model_type"] params["passed_data"]["model_type"] = model_type + if "var_model" not in runtimeparams.keys(): var_model = "sensor.power_load_no_var_loads" else: var_model = runtimeparams["var_model"] params["passed_data"]["var_model"] = var_model + if "var_model_in_kw" not in runtimeparams.keys(): + var_model_in_kw = False + else: + var_model_in_kw = eval(str(runtimeparams["var_model_in_kw"]).capitalize()) + params["passed_data"]["var_model_in_kw"] = var_model_in_kw if "sklearn_model" not in runtimeparams.keys(): sklearn_model = "KNeighborsRegressor" else: @@ -673,7 +679,7 @@ def get_injection_dict_forecast_model_fit(df_fit_pred: pd.DataFrame, mlf: MLFore "

Plotting train/test forecast model results for " + mlf.model_type + "

" ) injection_dict["subsubtitle0"] = ( - "

Forecasting variable " + mlf.var_model + "

" + "

Forecasting variable " + mlf.var_model + " (sensor in kW: " + mlf.var_model_in_kw + ")

" ) injection_dict["figure_0"] = image_path_0 return injection_dict @@ -743,8 +749,11 @@ def build_params(params: dict, params_secrets: dict, options: dict, addon: int, params["retrieve_hass_conf"]["var_load"] = options.get( "sensor_power_load_no_var_loads", params["retrieve_hass_conf"]["var_load"] ) - params["retrieve_hass_conf"]["load_sensor_kw"] = options.get( - "load_sensor_kw", params["retrieve_hass_conf"]["load_sensor_kw"] + params["retrieve_hass_conf"]["var_PV_in_kw"] = options.get( + "var_PV_in_kw", params["retrieve_hass_conf"]["var_PV_in_kw"] + ) + params["retrieve_hass_conf"]["var_load_in_kw"] = options.get( + "var_load_in_kw", params["retrieve_hass_conf"]["var_load_in_kw"] ) params["retrieve_hass_conf"]["load_negative"] = options.get( "load_negative", params["retrieve_hass_conf"]["load_negative"] diff --git a/tests/test_command_line_utils.py b/tests/test_command_line_utils.py index 91aa9f00..1cd1d4b9 100644 --- a/tests/test_command_line_utils.py +++ b/tests/test_command_line_utils.py @@ -279,6 +279,7 @@ def test_forecast_model_fit_predict_tune(self): "days_to_retrieve": 20, "model_type": "load_forecast", "var_model": "sensor.power_load_no_var_loads", + "var_model_in_kw": False, "sklearn_model": "KNeighborsRegressor", "num_lags": 48, "split_date_delta": '48h', @@ -457,6 +458,7 @@ def test_main_forecast_model_fit(self): "days_to_retrieve": 20, "model_type": "load_forecast", "var_model": "sensor.power_load_no_var_loads", + "var_model_in_kw": False, "sklearn_model": "KNeighborsRegressor", "num_lags": 48, "split_date_delta": '48h', @@ -479,6 +481,7 @@ def test_main_forecast_model_predict(self): "days_to_retrieve": 20, "model_type": "load_forecast", "var_model": "sensor.power_load_no_var_loads", + "var_model_in_kw": False, "sklearn_model": "KNeighborsRegressor", "num_lags": 48, "split_date_delta": "48h", @@ -501,6 +504,7 @@ def test_main_forecast_model_tune(self): "days_to_retrieve": 20, "model_type": "load_forecast", "var_model": "sensor.power_load_no_var_loads", + "var_model_in_kw": False, "sklearn_model": "KNeighborsRegressor", "num_lags": 48, "split_date_delta": "48h", diff --git a/tests/test_forecast.py b/tests/test_forecast.py index 946e33b4..5114cdc3 100644 --- a/tests/test_forecast.py +++ b/tests/test_forecast.py @@ -62,8 +62,9 @@ def setUp(self): else: self.days_list = utils.get_days_list(self.retrieve_hass_conf['days_to_retrieve']) self.var_list = [self.retrieve_hass_conf['var_load'], self.retrieve_hass_conf['var_PV']] + self.sensors_in_kw = [self.retrieve_hass_conf['var_load_in_kw'], self.retrieve_hass_conf['var_PV_in_kw']] self.rh.get_data(self.days_list, self.var_list, - minimal_response=False, significant_changes_only=False, load_sensor_kw = self.retrieve_hass_conf['load_sensor_kw']) + minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw) self.rh.prepare_data(self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'], set_zero_min = self.retrieve_hass_conf['set_zero_min'], var_replace_zero = self.retrieve_hass_conf['var_replace_zero'], @@ -220,8 +221,9 @@ def test_get_forecasts_with_lists(self): else: days_list = utils.get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] + sensors_in_kw = [retrieve_hass_conf['var_load_in_kw'], retrieve_hass_conf['var_PV_in_kw']] rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']) + minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw) rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], set_zero_min = retrieve_hass_conf['set_zero_min'], var_replace_zero = retrieve_hass_conf['var_replace_zero'], @@ -357,8 +359,9 @@ def test_get_forecasts_with_lists_special_case(self): else: days_list = utils.get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] + sensors_in_kw = [retrieve_hass_conf['var_load_in_kw'], retrieve_hass_conf['var_PV_in_kw']] rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False, load_sensor_kw = retrieve_hass_conf['load_sensor_kw']) + minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw) rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], set_zero_min = retrieve_hass_conf['set_zero_min'], var_replace_zero = retrieve_hass_conf['var_replace_zero'], @@ -455,6 +458,7 @@ def test_get_load_forecast_mlforecaster(self): "days_to_retrieve": 20, "model_type": "load_forecast", "var_model": "sensor.power_load_no_var_loads", + "var_model_in_kw": False, "sklearn_model": "KNeighborsRegressor", "num_lags": 48 } @@ -467,6 +471,7 @@ def test_get_load_forecast_mlforecaster(self): data = copy.deepcopy(input_data_dict['df_input_data']) model_type = input_data_dict['params']['passed_data']['model_type'] var_model = input_data_dict['params']['passed_data']['var_model'] + # TODO: is var_model_in_kw necessary to pass to MLForecaster()? sklearn_model = input_data_dict['params']['passed_data']['sklearn_model'] num_lags = input_data_dict['params']['passed_data']['num_lags'] mlf = MLForecaster(data, model_type, var_model, sklearn_model, num_lags, emhass_conf, logger) diff --git a/tests/test_machine_learning_forecaster.py b/tests/test_machine_learning_forecaster.py index 6d5d3787..8ee5fc50 100644 --- a/tests/test_machine_learning_forecaster.py +++ b/tests/test_machine_learning_forecaster.py @@ -56,6 +56,7 @@ def setUp(self): "days_to_retrieve": 20, "model_type": "load_forecast", "var_model": "sensor.power_load_no_var_loads", + "var_model_in_kw": False, "sklearn_model": "KNeighborsRegressor", "num_lags": 48 } @@ -68,6 +69,7 @@ def setUp(self): data = copy.deepcopy(self.input_data_dict['df_input_data']) model_type = self.input_data_dict['params']['passed_data']['model_type'] var_model = self.input_data_dict['params']['passed_data']['var_model'] + # var_model_in_kw not necessary if data is read from file, which should always be stored in W sklearn_model = self.input_data_dict['params']['passed_data']['sklearn_model'] num_lags = self.input_data_dict['params']['passed_data']['num_lags'] self.mlf = MLForecaster(data, model_type, var_model, sklearn_model, num_lags, emhass_conf, logger) diff --git a/tests/test_optimization.py b/tests/test_optimization.py index e2f24e8a..4807662c 100644 --- a/tests/test_optimization.py +++ b/tests/test_optimization.py @@ -44,8 +44,9 @@ def setUp(self): else: self.days_list = get_days_list(self.retrieve_hass_conf['days_to_retrieve']) self.var_list = [self.retrieve_hass_conf['var_load'], self.retrieve_hass_conf['var_PV']] + self.sensors_in_kw = [self.retrieve_hass_conf['var_load_in_kw'], self.retrieve_hass_conf['var_PV_in_kw']] self.rh.get_data(self.days_list, self.var_list, - minimal_response=False, significant_changes_only=False,load_sensor_kw=self.retrieve_hass_conf['load_sensor_kw']) + minimal_response=False, significant_changes_only=False,sensor_in_kw_list=sensors_in_kw) self.rh.prepare_data(self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'], set_zero_min = self.retrieve_hass_conf['set_zero_min'], var_replace_zero = self.retrieve_hass_conf['var_replace_zero'], diff --git a/tests/test_retrieve_hass.py b/tests/test_retrieve_hass.py index 0fc3c1d3..0a00532e 100644 --- a/tests/test_retrieve_hass.py +++ b/tests/test_retrieve_hass.py @@ -46,8 +46,9 @@ def setUp(self): else: self.days_list = get_days_list(self.retrieve_hass_conf['days_to_retrieve']) self.var_list = [self.retrieve_hass_conf['var_load'], self.retrieve_hass_conf['var_PV']] + self.sensors_in_kw = [self.retrieve_hass_conf['var_load_in_kw'], self.retrieve_hass_conf['var_PV_in_kw']] self.rh.get_data(self.days_list, self.var_list, - minimal_response=False, significant_changes_only=False, load_sensor_kw=self.retrieve_hass_conf['load_sensor_kw']) + minimal_response=False, significant_changes_only=False, self.sensor_in_kw_list=sensors_in_kw) if save_data_to_file: with open(emhass_conf['data_path'] / 'test_df_final.pkl', 'wb') as outp: pickle.dump((self.rh.df_final, self.days_list, self.var_list), @@ -95,9 +96,25 @@ def test_yaml_parse_wab_server(self): def test_get_data_failed(self): days_list = get_days_list(1) var_list = [self.retrieve_hass_conf['var_load']] - response = self.rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, load_sensor_kw=self.retrieve_hass_conf['load_sensor_kw']) + sensors_in_kw = [self.retriev_hass_conf['var_load_in_kw']] + response = self.rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw) self.assertFalse(response) + def test_get_data_in_kw(self): + days_list = get_days_list(1) + var_list = [self.retrieve_hass_conf['var_load']] + r1 = self.rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=[True]) + r2 = self.rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=[False]) + # test that r1 df entry is 1000x same r2 df entry + for i,row in r1.iterrows(): + value_r1=row[1] + value_r2=r2.loc[i,1] + if value_r1>0: + if value_r2 == value_r1 * 1000: + self.assertTrue(r1) + break + self.assertFalse(r1) + def test_get_data_mock(self): with requests_mock.mock() as m: days_list = get_days_list(1) @@ -107,8 +124,7 @@ def test_get_data_mock(self): m.get(self.retrieve_hass_conf['hass_url'], json=data.json()) self.rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, - test_url=self.retrieve_hass_conf['hass_url'] - load_sensor_kw=self.retrieve_hass_conf['load_sensor_kw']) + test_url=self.retrieve_hass_conf['hass_url']) self.assertIsInstance(self.rh.df_final, type(pd.DataFrame())) self.assertIsInstance(self.rh.df_final.index, pd.core.indexes.datetimes.DatetimeIndex) self.assertIsInstance(self.rh.df_final.index.dtype, pd.core.dtypes.dtypes.DatetimeTZDtype) From fa0bd3cbe33548dbc8546e74ab3c536c8b8e2c0d Mon Sep 17 00:00:00 2001 From: Pedro Venda Date: Tue, 28 May 2024 14:21:37 +0100 Subject: [PATCH 3/3] syntax error fixed --- tests/test_retrieve_hass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_retrieve_hass.py b/tests/test_retrieve_hass.py index 0a00532e..2da32325 100644 --- a/tests/test_retrieve_hass.py +++ b/tests/test_retrieve_hass.py @@ -48,7 +48,7 @@ def setUp(self): self.var_list = [self.retrieve_hass_conf['var_load'], self.retrieve_hass_conf['var_PV']] self.sensors_in_kw = [self.retrieve_hass_conf['var_load_in_kw'], self.retrieve_hass_conf['var_PV_in_kw']] self.rh.get_data(self.days_list, self.var_list, - minimal_response=False, significant_changes_only=False, self.sensor_in_kw_list=sensors_in_kw) + minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw) if save_data_to_file: with open(emhass_conf['data_path'] / 'test_df_final.pkl', 'wb') as outp: pickle.dump((self.rh.df_final, self.days_list, self.var_list),