Imp - Added primitive implementation of now/current values forecast c…

…orrection

src/emhass/command_line.py

@@ -43,24 +43,23 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str,
     # Treat runtimeparams
     params, optim_conf = utils.treat_runtimeparams(runtimeparams, params, retrieve_hass_conf, 
                                                    optim_conf, plant_conf, set_type, logger)
-    # Initialize objects
-    if get_data_from_file:
-        with open(pathlib.Path(base_path+'/data/test_df_final.pkl'), 'rb') as inp:
-            df_final, days_list, var_list = pickle.load(inp)
-    else:
-        days_list = utils.get_days_list(retrieve_hass_conf['days_to_retrieve'])
-        var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']]
     # Define main objects
     rh = retrieve_hass(retrieve_hass_conf['hass_url'], retrieve_hass_conf['long_lived_token'], 
                        retrieve_hass_conf['freq'], retrieve_hass_conf['time_zone'], 
                        params, base_path, logger, get_data_from_file=get_data_from_file)
     fcst = forecast(retrieve_hass_conf, optim_conf, plant_conf,
                     params, base_path, logger, get_data_from_file=get_data_from_file)
     opt = optimization(retrieve_hass_conf, optim_conf, plant_conf, 
-                       fcst.var_load_cost, fcst.var_prod_price, days_list, 
+                       fcst.var_load_cost, fcst.var_prod_price, 
                        costfun, base_path, logger)
     # Perform setup based on type of action
     if set_type == "perfect-optim":
+        if get_data_from_file:
+            with open(pathlib.Path(base_path+'/data/test_df_final.pkl'), 'rb') as inp:
+                df_final, days_list, var_list = pickle.load(inp)
+        else:
+            days_list = utils.get_days_list(retrieve_hass_conf['days_to_retrieve'])
+            var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']]
         # Retrieve data from hass
         if get_data_from_file:
             df_input_data = copy.deepcopy(df_final)
@@ -74,7 +73,7 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str,
             df_input_data = rh.df_final.copy()
         # What we don't need for this type of action
         P_PV_forecast, P_load_forecast, df_input_data_dayahead = None, None, None
-    elif set_type == "dayahead-optim" or set_type == "naive-mpc-optim":
+    elif set_type == "dayahead-optim":
         # Get PV and load forecasts
         df_weather = fcst.get_weather_forecast(method=optim_conf['weather_forecast_method'])
         P_PV_forecast = fcst.get_power_from_weather(df_weather)
@@ -87,10 +86,40 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str,
             prediction_horizon = params['passed_data']['prediction_horizon']
             df_input_data_dayahead = copy.deepcopy(df_input_data_dayahead)[df_input_data_dayahead.index[0]:df_input_data_dayahead.index[prediction_horizon-1]]
         # What we don't need for this type of action
-        df_input_data = None
+        df_input_data, days_list = None, None
+    elif set_type == "naive-mpc-optim":
+        if get_data_from_file:
+            with open(pathlib.Path(base_path+'/data/test_df_final.pkl'), 'rb') as inp:
+                df_final, days_list, var_list = pickle.load(inp)
+        else:
+            days_list = utils.get_days_list(1)
+            var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']]
+        # Retrieve data from hass
+        if get_data_from_file:
+            df_input_data = copy.deepcopy(df_final)
+        else:
+            rh.get_data(days_list, var_list,
+                        minimal_response=False, significant_changes_only=False)
+            rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'],
+                            set_zero_min = retrieve_hass_conf['set_zero_min'], 
+                            var_replace_zero = retrieve_hass_conf['var_replace_zero'], 
+                            var_interp = retrieve_hass_conf['var_interp'])
+            df_input_data = rh.df_final.copy()
+        # Get PV and load forecasts
+        df_weather = fcst.get_weather_forecast(method=optim_conf['weather_forecast_method'])
+        P_PV_forecast = fcst.get_power_from_weather(df_weather, set_mix_forecast=True, df_now=df_input_data)
+        P_load_forecast = fcst.get_load_forecast(method=optim_conf['load_forecast_method'], set_mix_forecast=True, df_now=df_input_data)
+        df_input_data_dayahead = pd.concat([P_PV_forecast, P_load_forecast], axis=1)
+        df_input_data_dayahead = utils.set_df_index_freq(df_input_data_dayahead)
+        df_input_data_dayahead.columns = ['P_PV_forecast', 'P_load_forecast']
+        params = json.loads(params)
+        if 'prediction_horizon' in params['passed_data'] and params['passed_data']['prediction_horizon'] is not None:
+            prediction_horizon = params['passed_data']['prediction_horizon']
+            df_input_data_dayahead = copy.deepcopy(df_input_data_dayahead)[df_input_data_dayahead.index[0]:df_input_data_dayahead.index[prediction_horizon-1]]
     elif set_type == "publish-data":
         df_input_data, df_input_data_dayahead = None, None
         P_PV_forecast, P_load_forecast = None, None
+        days_list = None
     else:
         logger.error("The passed action argument and hence the set_type parameter for setup is not valid")
 
@@ -106,7 +135,8 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str,
         'P_PV_forecast': P_PV_forecast,
         'P_load_forecast': P_load_forecast,
         'costfun': costfun,
-        'params': params
+        'params': params,
+        'days_list': days_list
     }
     return input_data_dict
 
@@ -134,7 +164,7 @@ def perfect_forecast_optim(input_data_dict: dict, logger: logging.Logger,
         method=input_data_dict['fcst'].optim_conf['load_cost_forecast_method'])
     df_input_data = input_data_dict['fcst'].get_prod_price_forecast(
         df_input_data, method=input_data_dict['fcst'].optim_conf['prod_price_forecast_method'])
-    opt_res = input_data_dict['opt'].perform_perfect_forecast_optim(df_input_data)
+    opt_res = input_data_dict['opt'].perform_perfect_forecast_optim(df_input_data, input_data_dict['days_list'])
     # Save CSV file for analysis
     if save_data_to_file:
         filename = 'opt_res_perfect_optim_'+input_data_dict['costfun']

src/emhass/forecast.py

@@ -275,14 +275,26 @@ def cloud_cover_to_irradiance(self, cloud_cover: pd.Series,
         irrads = pd.DataFrame({'ghi': ghi, 'dni': dni, 'dhi': dhi}).fillna(0)
         return irrads
 
-    def get_power_from_weather(self, df_weather: pd.DataFrame) -> pd.Series:
+    @staticmethod
+    def get_mix_forecast(df_now, df_forecast, alpha, beta, col):
+        first_fcst = alpha*df_forecast.iloc[0] + beta*df_now[col].iloc[-1]
+        df_forecast.iloc[0] = first_fcst
+        return df_forecast
+
+    def get_power_from_weather(self, df_weather: pd.DataFrame, 
+                               set_mix_forecast:Optional[bool] = False,
+                               df_now:Optional[pd.DataFrame] = pd.DataFrame()) -> pd.Series:
         """
         Convert wheater forecast data into electrical power.
         
         :param df_weather: The DataFrame containing the weather forecasted data. \
             This DF should be generated by the 'get_weather_forecast' method or at \
             least contain the same columns names filled with proper data.
         :type df_weather: pd.DataFrame
+        :param set_mix_forecast: Use a mixed forcast strategy to integra now/current values.
+        :type set_mix_forecast: Bool, optional
+        :param df_now: The DataFrame containing the now/current data.
+        :type df_now: pd.DataFrame
         :return: The DataFrame containing the electrical power in Watts
         :rtype: pd.DataFrame
 
@@ -335,6 +347,9 @@ def get_power_from_weather(self, df_weather: pd.DataFrame) -> pd.Series:
                 # Extracting results for AC power
                 P_PV_forecast = mc.results.ac
 
+        if set_mix_forecast:
+            P_PV_forecast = forecast.get_mix_forecast(df_now, P_PV_forecast, 0.5, 0.5, 'sensor.power_photovoltaics')
+
         return P_PV_forecast
 
     def get_forecast_days_csv(self, timedelta_days: Optional[int] = 1) -> pd.date_range:
@@ -419,7 +434,8 @@ def get_forecast_out_from_csv(self, df_final: pd.DataFrame, forecast_dates_csv:
         return forecast_out
 
     def get_load_forecast(self, days_min_load_forecast: Optional[int] = 3, method: Optional[str] = 'naive',
-                          csv_path: Optional[str] = "/data/data_load_forecast.csv") -> pd.Series:
+                          csv_path: Optional[str] = "/data/data_load_forecast.csv",
+                          set_mix_forecast:Optional[bool] = False, df_now:Optional[pd.DataFrame] = pd.DataFrame()) -> pd.Series:
         """
         Get and generate the load forecast data.
         
@@ -432,6 +448,10 @@ def get_load_forecast(self, days_min_load_forecast: Optional[int] = 3, method: O
         :param csv_path: The path to the CSV file used when method = 'csv', \
             defaults to "/data/data_load_forecast.csv"
         :type csv_path: str, optional
+        param set_mix_forecast: Use a mixed forcast strategy to integra now/current values.
+        :type set_mix_forecast: Bool, optional
+        :param df_now: The DataFrame containing the now/current data.
+        :type df_now: pd.DataFrame, optional
         :return: The DataFrame containing the electrical load power in Watts
         :rtype: pd.DataFrame
 
@@ -509,8 +529,12 @@ def get_load_forecast(self, days_min_load_forecast: Optional[int] = 3, method: O
 
         else:
             self.logger.error("Passed method is not valid")
+
+        P_Load_forecast = copy.deepcopy(forecast_out['yhat'])
+        if set_mix_forecast:
+            P_Load_forecast = forecast.get_mix_forecast(df_now, P_Load_forecast, 0.5, 0.5, 'sensor.power_load_no_var_loads')
 
-        return forecast_out['yhat']
+        return P_Load_forecast
 
     def get_load_cost_forecast(self, df_final: pd.DataFrame, method: Optional[str] = 'hp_hc_periods',
                                csv_path: Optional[str] = "/data/data_load_cost_forecast.csv") -> pd.DataFrame:

src/emhass/optimization.py

@@ -25,7 +25,7 @@ class optimization:
     """
 
     def __init__(self, retrieve_hass_conf: dict, optim_conf: dict, plant_conf: dict, 
-                 var_load_cost: str, var_prod_price: str, days_list: pd.date_range, 
+                 var_load_cost: str, var_prod_price: str, 
                  costfun: str, config_path: str, logger: logging.Logger, 
                  opt_time_delta: Optional[int] = 24) -> None:
         """
@@ -43,10 +43,6 @@ def __init__(self, retrieve_hass_conf: dict, optim_conf: dict, plant_conf: dict,
         :type var_load_cost: str
         :param var_prod_price: The column name for the unit power production price.
         :type var_prod_price: str
-        :param days_list: A list of the days of data that will be retrieved from \
-            hass and used for the optimization task. We will retrieve data from \
-            now and up to days_to_retrieve days
-        :type days_list: list
         :param costfun: The type of cost function to use for optimization problem
         :type costfun: str
         :param config_path: The path to the yaml configuration file
@@ -62,7 +58,6 @@ def __init__(self, retrieve_hass_conf: dict, optim_conf: dict, plant_conf: dict,
         self.retrieve_hass_conf = retrieve_hass_conf
         self.optim_conf = optim_conf
         self.plant_conf = plant_conf
-        self.days_list = days_list
         self.freq = self.retrieve_hass_conf['freq']
         self.time_zone = self.retrieve_hass_conf['time_zone']
         self.timeStep = self.freq.seconds/3600 # in hours
@@ -433,19 +428,23 @@ def perform_optimization(self, data_opt: pd.DataFrame, P_PV: np.array, P_load: n
 
         return opt_tp
 
-    def perform_perfect_forecast_optim(self, df_input_data: pd.DataFrame) -> pd.DataFrame:
+    def perform_perfect_forecast_optim(self, df_input_data: pd.DataFrame, days_list: pd.date_range) -> pd.DataFrame:
         """
         Perform an optimization on historical data (perfectly known PV production).
         
         :param df_input_data: A DataFrame containing all the input data used for \
             the optimization, notably photovoltaics and load consumption powers.
         :type df_input_data: pandas.DataFrame
+        :param days_list: A list of the days of data that will be retrieved from \
+            hass and used for the optimization task. We will retrieve data from \
+            now and up to days_to_retrieve days
+        :type days_list: list
         :return: opt_res: A DataFrame containing the optimization results
         :rtype: pandas.DataFrame
 
         """
         self.logger.info("Perform optimization for perfect forecast scenario")
-        self.days_list_tz = self.days_list.tz_convert(self.time_zone).round(self.freq)[:-1] # Converted to tz and without the current day (today)
+        self.days_list_tz = days_list.tz_convert(self.time_zone).round(self.freq)[:-1] # Converted to tz and without the current day (today)
         self.opt_res = pd.DataFrame()
         for day in self.days_list_tz:
             self.logger.info("Solving for day: "+str(day.day)+"-"+str(day.month)+"-"+str(day.year))

tests/test_command_line_utils.py

@@ -126,7 +126,6 @@ def test_set_input_data_dict(self):
         input_data_dict = set_input_data_dict(config_path, base_path, costfun, self.params_json, self.runtimeparams_json, 
                                               action, logger, get_data_from_file=True)
         self.assertIsInstance(input_data_dict, dict)
-        self.assertTrue(input_data_dict['df_input_data'] == None)
         self.assertIsInstance(input_data_dict['df_input_data_dayahead'], pd.DataFrame)
         self.assertTrue(input_data_dict['df_input_data_dayahead'].index.freq is not None)
         self.assertTrue(input_data_dict['df_input_data_dayahead'].isnull().sum().sum()==0)
@@ -148,7 +147,6 @@ def test_set_input_data_dict(self):
         input_data_dict = set_input_data_dict(config_path, base_path, costfun, params_json, runtimeparams_json, 
                                               action, logger, get_data_from_file=True)
         self.assertIsInstance(input_data_dict, dict)
-        self.assertTrue(input_data_dict['df_input_data'] == None)
         self.assertIsInstance(input_data_dict['df_input_data_dayahead'], pd.DataFrame)
         self.assertTrue(input_data_dict['df_input_data_dayahead'].index.freq is not None)
         self.assertTrue(input_data_dict['df_input_data_dayahead'].isnull().sum().sum()==0)

tests/test_optimization.py

@@ -54,15 +54,15 @@ def setUp(self):
         self.costfun = 'profit'
         self.opt = optimization(self.retrieve_hass_conf, self.optim_conf, self.plant_conf, 
                                 self.fcst.var_load_cost, self.fcst.var_prod_price,  
-                                self.days_list, self.costfun, root, logger)
+                                self.costfun, root, logger)
         self.df_input_data = self.fcst.get_load_cost_forecast(self.df_input_data)
         self.df_input_data = self.fcst.get_prod_price_forecast(self.df_input_data)
         self.input_data_dict = {
             'retrieve_hass_conf': retrieve_hass_conf,
         }
 
     def test_perform_perfect_forecast_optim(self):
-        self.opt_res = self.opt.perform_perfect_forecast_optim(self.df_input_data)
+        self.opt_res = self.opt.perform_perfect_forecast_optim(self.df_input_data, self.days_list)
         self.assertIsInstance(self.opt_res, type(pd.DataFrame()))
         self.assertIsInstance(self.opt_res.index, pd.core.indexes.datetimes.DatetimeIndex)
         self.assertIsInstance(self.opt_res.index.dtype, pd.core.dtypes.dtypes.DatetimeTZDtype)
@@ -87,7 +87,7 @@ def test_perform_dayahead_forecast_optim(self):
         self.optim_conf.update({'set_use_battery': True})
         self.opt = optimization(self.retrieve_hass_conf, self.optim_conf, self.plant_conf, 
                                 self.fcst.var_load_cost, self.fcst.var_prod_price,  
-                                self.days_list, self.costfun, root, logger)
+                                self.costfun, root, logger)
         self.opt_res_dayahead = self.opt.perform_dayahead_forecast_optim(
             self.df_input_data_dayahead, self.P_PV_forecast, self.P_load_forecast)
         self.assertIsInstance(self.opt_res_dayahead, type(pd.DataFrame()))
@@ -102,7 +102,7 @@ def test_perform_naive_mpc_optim(self):
         self.optim_conf.update({'set_use_battery': True})
         self.opt = optimization(self.retrieve_hass_conf, self.optim_conf, self.plant_conf, 
                                 self.fcst.var_load_cost, self.fcst.var_prod_price,  
-                                self.days_list, self.costfun, root, logger)
+                                self.costfun, root, logger)
         prediction_horizon = 10
         soc_init = 0.4
         soc_final = 0.6