diff --git a/config_emhass.yaml b/config_emhass.yaml index b73e6d99..9c7e3c8d 100644 --- a/config_emhass.yaml +++ b/config_emhass.yaml @@ -4,7 +4,9 @@ retrieve_hass_conf: freq: 30 # The time step to resample retrieved data from hass in minutes days_to_retrieve: 2 # We will retrieve data from now and up to days_to_retrieve days var_PV: 'sensor.power_photovoltaics' # Photovoltaic produced power sensor in Watts + var_PV_in_kw: False # Photovoltaic sensor in kW rather than W var_load: 'sensor.power_load_no_var_loads' # Household power consumption sensor in Watts (deferrable loads should be substracted) + var_load_in_kw: False # load_no_var_loads in kW rather than W load_negative: False # Set to True if the retrived load variable is negative by convention set_zero_min: True # A special treatment for a minimum value saturation to zero. Values below zero are replaced by nans var_replace_zero: # A list of retrived variables that we would want to replace nans with zeros diff --git a/docs/config.md b/docs/config.md index ef46ad20..6bbaf131 100644 --- a/docs/config.md +++ b/docs/config.md @@ -15,7 +15,9 @@ These are the parameters that we will need to define to retrieve data from Home - `freq`: The time step to resample retrieved data from hass. This parameter is given in minutes. It should not be defined too low or you will run into memory problems when defining the Linear Programming optimization. Defaults to 30. - `days_to_retrieve`: We will retrieve data from now and up to days_to_retrieve days. Defaults to 2. - `var_PV`: This is the name of the photovoltaic produced power sensor in Watts from Home Assistant. For example: 'sensor.power_photovoltaics'. +- `var_PV_in_kw`: Set this parameter to True if the PV sensor is in kW rather than W. Defaults to False. - `var_load`: The name of the household power consumption sensor in Watts from Home Assistant. The deferrable loads that we will want to include in the optimization problem should be substracted from this sensor in HASS. For example: 'sensor.power_load_no_var_loads' +- `var_load_in_kw`: Set this parameter to True if the load sensor is in kW rather than W. Defaults to False. - `load_negative`: Set this parameter to True if the retrived load variable is negative by convention. Defaults to False. - `set_zero_min`: Set this parameter to True to give a special treatment for a minimum value saturation to zero for power consumption data. Values below zero are replaced by nans. Defaults to True. - `var_replace_zero`: The list of retrieved variables that we would want to replace nans (if they exist) with zeros. For example: diff --git a/docs/differences.md b/docs/differences.md index 68fbf5f1..88d76227 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -20,6 +20,8 @@ See bellow for a list of associations between the parameters from `config_emhass | retrieve_hass_conf | var_PV | sensor_power_photovoltaics | | | retrieve_hass_conf | var_load | sensor_power_load_no_var_loads | | | retrieve_hass_conf | load_negative | load_negative | | +| retrieve_hass_conf | var_PV_in_kw | var_PV_in_kw | | +| retrieve_hass_conf | var_load_in_kw | var_load_in_kw | | | retrieve_hass_conf | set_zero_min | set_zero_min | | | retrieve_hass_conf | method_ts_round | method_ts_round | | | params_secrets | solcast_api_key | optional_solcast_api_key | | diff --git a/docs/mlforecaster.md b/docs/mlforecaster.md index 1de5d38c..bf32e178 100644 --- a/docs/mlforecaster.md +++ b/docs/mlforecaster.md @@ -26,6 +26,8 @@ The minimum number of `days_to_retrieve` is hard coded to 9 by default. But it i - `var_model`: the name of the sensor to retrieve data from Home Assistant. Example: `sensor.power_load_no_var_loads`. +- `var_model_in_kw`: whether the sensor is in kW. Example: `False`. + - `sklearn_model`: the `scikit-learn` model that will be used. For now only this options are possible: `LinearRegression`, `ElasticNet` and `KNeighborsRegressor`. - `num_lags`: the number of auto-regression lags to consider. A good starting point is to fix this as one day. For example if your time step is 30 minutes, then fix this to 48, if the time step is 1 hour the fix this to 24 and so on. diff --git a/options.json b/options.json index b0b55076..e56e0bd7 100644 --- a/options.json +++ b/options.json @@ -22,6 +22,8 @@ "weight_battery_charge": 1.0, "sensor_power_photovoltaics": "sensor.power_photovoltaics", "sensor_power_load_no_var_loads": "sensor.power_load_no_var_loads", + "var_PV_in_kw": false, + "var_load_in_kw": false, "load_negative": false, "set_zero_min": true, "number_of_deferrable_loads": 2, diff --git a/scripts/load_clustering.py b/scripts/load_clustering.py index db92ea2e..c8af90ff 100644 --- a/scripts/load_clustering.py +++ b/scripts/load_clustering.py @@ -47,6 +47,7 @@ days_to_retrieve = 240 model_type = "load_clustering" var_model = "sensor.power_load_positive" + var_model_in_kw = False data_path = emhass_conf['data_path'] / str('data_train_'+model_type+'.pkl') params = None @@ -65,7 +66,8 @@ days_list = get_days_list(days_to_retrieve) var_list = [var_model] - rh.get_data(days_list, var_list) + sensors_in_kw = [var_model_in_kw] + rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw) with open(data_path, 'wb') as fid: pickle.dump((rh.df_final, var_model), fid, pickle.HIGHEST_PROTOCOL) @@ -144,4 +146,4 @@ # data_lag['cluster_group_tslearn_sdtw'] = y_pred # fig = px.scatter(data_lag, x='power_load y(t)', y='power_load y(t+1)', color='cluster_group_tslearn_sdtw', template=template) - # fig.show() \ No newline at end of file + # fig.show() diff --git a/scripts/load_forecast_sklearn.py b/scripts/load_forecast_sklearn.py index 82f5b3b3..4bcaabda 100644 --- a/scripts/load_forecast_sklearn.py +++ b/scripts/load_forecast_sklearn.py @@ -54,6 +54,7 @@ def neg_r2_score(y_true, y_pred): days_to_retrieve = 240 model_type = "load_forecast" var_model = "sensor.power_load_no_var_loads" + var_model_in_kw = False sklearn_model = "KNeighborsRegressor" num_lags = 48 @@ -74,7 +75,8 @@ def neg_r2_score(y_true, y_pred): days_list = get_days_list(days_to_retrieve) var_list = [var_model] - rh.get_data(days_list, var_list) + sensors_in_kw = [var_model_in_kw] + rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw) with open(data_path, 'wb') as fid: pickle.dump((rh.df_final, var_model), fid, pickle.HIGHEST_PROTOCOL) @@ -257,7 +259,8 @@ def neg_r2_score(y_true, y_pred): days_list = get_days_list(days_needed) var_model = retrieve_hass_conf['var_load'] var_list = [var_model] - rh.get_data(days_list, var_list) + sensor_in_kw_list = [retrieve_hass_conf['var_load_in_kw']] + rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list) data_last_window = copy.deepcopy(rh.df_final) data_last_window = add_date_features(data_last_window) @@ -275,4 +278,4 @@ def neg_r2_score(y_true, y_pred): fig.update_yaxes(title_text = "Power (W)") fig.update_xaxes(title_text = "Time") fig.show() - fig.write_image(emhass_conf['root_path'] / "docs/images/load_forecast_production.svg", width=1080, height=0.8*1080) \ No newline at end of file + fig.write_image(emhass_conf['root_path'] / "docs/images/load_forecast_production.svg", width=1080, height=0.8*1080) diff --git a/scripts/optim_results_analysis.py b/scripts/optim_results_analysis.py index 173c7045..8da7d5ae 100644 --- a/scripts/optim_results_analysis.py +++ b/scripts/optim_results_analysis.py @@ -130,4 +130,4 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, print(opt_res_dah) if save_html: - opt_res_dah.to_html('opt_res_dah.html') \ No newline at end of file + opt_res_dah.to_html('opt_res_dah.html') diff --git a/scripts/special_config_analysis.py b/scripts/special_config_analysis.py index ace61ab8..497d0419 100644 --- a/scripts/special_config_analysis.py +++ b/scripts/special_config_analysis.py @@ -185,4 +185,4 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, fig_res_mpc.layout.template = template fig_res_mpc.update_yaxes(title_text = "Powers (W)") fig_res_mpc.update_xaxes(title_text = "Time") - fig_res_mpc.show() \ No newline at end of file + fig_res_mpc.show() diff --git a/scripts/use_cases_analysis.py b/scripts/use_cases_analysis.py index 0030784e..b5f6d620 100644 --- a/scripts/use_cases_analysis.py +++ b/scripts/use_cases_analysis.py @@ -55,8 +55,9 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, params, emhass_conf, logger) days_list = get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] + sensors_in_kw = [retrieve_hass_conf['var_load_in_kw'], retrieve_hass_conf['var_PV_in_kw']] rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False) + minimal_response=False, significant_changes_only=False,sensor_in_kw_list=sensors_in_kw) rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], set_zero_min = retrieve_hass_conf['set_zero_min'], var_replace_zero = retrieve_hass_conf['var_replace_zero'], @@ -177,4 +178,4 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf, width=1080, height=0.8*1080) print("System with: PV, Battery, two deferrable loads, dayahead optimization, profit >> total cost function sum: "+\ - str(opt_res_dah['cost_profit'].sum())) \ No newline at end of file + str(opt_res_dah['cost_profit'].sum())) diff --git a/src/emhass/command_line.py b/src/emhass/command_line.py index 7bb455b3..f5153bfa 100644 --- a/src/emhass/command_line.py +++ b/src/emhass/command_line.py @@ -81,7 +81,9 @@ def set_input_data_dict(emhass_conf: dict, costfun: str, retrieve_hass_conf["days_to_retrieve"]) var_list = [retrieve_hass_conf["var_load"], retrieve_hass_conf["var_PV"]] - if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False): + sensors_in_kw = [retrieve_hass_conf["var_load_in_kw"], + retrieve_hass_conf["var_PV_in_kw"]] + if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw): return False if not rh.prepare_data(retrieve_hass_conf["var_load"], load_negative=retrieve_hass_conf["load_negative"], @@ -129,7 +131,9 @@ def set_input_data_dict(emhass_conf: dict, costfun: str, days_list = utils.get_days_list(1) var_list = [retrieve_hass_conf["var_load"], retrieve_hass_conf["var_PV"]] - if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False): + sensors_in_kw = [retrieve_hass_conf["var_load_in_kw"], + retrieve_hass_conf["var_PV_in_kw"]] + if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw): return False if not rh.prepare_data(retrieve_hass_conf["var_load"], load_negative=retrieve_hass_conf["load_negative"], @@ -165,6 +169,7 @@ def set_input_data_dict(emhass_conf: dict, costfun: str, days_to_retrieve = params["passed_data"]["days_to_retrieve"] model_type = params["passed_data"]["model_type"] var_model = params["passed_data"]["var_model"] + var_model_in_kw = params["passed_data"]["var_model_in_kw"] if get_data_from_file: days_list = None filename = 'data_train_'+model_type+'.pkl' @@ -175,7 +180,8 @@ def set_input_data_dict(emhass_conf: dict, costfun: str, else: days_list = utils.get_days_list(days_to_retrieve) var_list = [var_model] - if not rh.get_data(days_list, var_list): + sensors_in_kw = [var_model_in_kw] + if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw): return False df_input_data = rh.df_final.copy() elif set_type == "regressor-model-fit" or set_type == "regressor-model-predict": @@ -404,6 +410,7 @@ def forecast_model_fit(input_data_dict: dict, logger: logging.Logger, data = copy.deepcopy(input_data_dict['df_input_data']) model_type = input_data_dict['params']['passed_data']['model_type'] var_model = input_data_dict['params']['passed_data']['var_model'] + var_model_in_kw = input_data_dict['params']['passed_data']['var_model_in_kw'] sklearn_model = input_data_dict['params']['passed_data']['sklearn_model'] num_lags = input_data_dict['params']['passed_data']['num_lags'] split_date_delta = input_data_dict['params']['passed_data']['split_date_delta'] diff --git a/src/emhass/forecast.py b/src/emhass/forecast.py index c0c74a85..03edd8de 100644 --- a/src/emhass/forecast.py +++ b/src/emhass/forecast.py @@ -643,7 +643,8 @@ def get_load_forecast(self, days_min_load_forecast: Optional[int] = 3, method: O self.var_load_new = self.var_load+'_positive' else: days_list = get_days_list(days_min_load_forecast) - if not rh.get_data(days_list, var_list): + sensors_in_kw = [self.retrieve_hass_conf['var_load_in_kw']] + if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw): return False if not rh.prepare_data( self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'], diff --git a/src/emhass/retrieve_hass.py b/src/emhass/retrieve_hass.py index 476a7c9b..9511a8ff 100644 --- a/src/emhass/retrieve_hass.py +++ b/src/emhass/retrieve_hass.py @@ -67,7 +67,7 @@ def __init__(self, hass_url: str, long_lived_token: str, freq: pd.Timedelta, def get_data(self, days_list: pd.date_range, var_list: list, minimal_response: Optional[bool] = False, significant_changes_only: Optional[bool] = False, - test_url: Optional[str] = "empty") -> None: + test_url: Optional[str] = "empty", sensor_in_kw_list: Optional[list] = []) -> None: r""" Retrieve the actual data from hass. @@ -84,6 +84,9 @@ def get_data(self, days_list: pd.date_range, var_list: list, :param significant_changes_only: Retrieve significant changes only \ using the hass restful API, defaults to False :type significant_changes_only: bool, optional + :param sensor_in_kw_list: Array of flags that determine whether the \ + var_list elements are in kW or W. False if undefined + :type sensor_in_kw_list: list, optional :return: The DataFrame populated with the retrieved data from hass :rtype: pandas.DataFrame @@ -178,16 +181,37 @@ def get_data(self, days_list: pd.date_range, var_list: list, if i == 0: # Defining the DataFrame container from_date = pd.to_datetime(df_raw['last_changed'], format="ISO8601").min() to_date = pd.to_datetime(df_raw['last_changed'], format="ISO8601").max() - ts = pd.to_datetime(pd.date_range(start=from_date, end=to_date, freq=self.freq), + ts = pd.to_datetime(pd.date_range(start=from_date, end=to_date, freq=self.freq), format='%Y-%d-%m %H:%M').round(self.freq, ambiguous='infer', nonexistent='shift_forward') df_day = pd.DataFrame(index = ts) - # Caution with undefined string data: unknown, unavailable, etc. - df_tp = ( - df_raw.copy()[["state"]] - .replace(["unknown", "unavailable", ""], np.nan) - .astype(float) - .rename(columns={"state": var}) - ) + # if load sensors are in kW, multiply the state entry by 1000 + if sensor_in_kw_list: + if sensor_in_kw_list[i]: + # Caution with undefined string data: unknown, unavailable, etc. + df_tp = ( + df_raw.copy()[["state"]] + .replace(["unknown", "unavailable", ""], np.nan) + .astype(float) + .rename(columns={"state": var}) + .mul(1000) + ) + else: + # Caution with undefined string data: unknown, unavailable, etc. + df_tp = ( + df_raw.copy()[["state"]] + .replace(["unknown", "unavailable", ""], np.nan) + .astype(float) + .rename(columns={"state": var}) + ) + else: + # Caution with undefined string data: unknown, unavailable, etc. + df_tp = ( + df_raw.copy()[["state"]] + .replace(["unknown", "unavailable", ""], np.nan) + .astype(float) + .rename(columns={"state": var}) + ) + # Setting index, resampling and concatenation df_tp.set_index( pd.to_datetime(df_raw["last_changed"], format="ISO8601"), diff --git a/src/emhass/utils.py b/src/emhass/utils.py index f32a136d..92cef398 100644 --- a/src/emhass/utils.py +++ b/src/emhass/utils.py @@ -325,11 +325,17 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic else: model_type = runtimeparams["model_type"] params["passed_data"]["model_type"] = model_type + if "var_model" not in runtimeparams.keys(): var_model = "sensor.power_load_no_var_loads" else: var_model = runtimeparams["var_model"] params["passed_data"]["var_model"] = var_model + if "var_model_in_kw" not in runtimeparams.keys(): + var_model_in_kw = False + else: + var_model_in_kw = eval(str(runtimeparams["var_model_in_kw"]).capitalize()) + params["passed_data"]["var_model_in_kw"] = var_model_in_kw if "sklearn_model" not in runtimeparams.keys(): sklearn_model = "KNeighborsRegressor" else: @@ -673,7 +679,7 @@ def get_injection_dict_forecast_model_fit(df_fit_pred: pd.DataFrame, mlf: MLFore "