diff --git a/.env.example b/.env.example index ec52b660..6fa99709 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,11 @@ SOLIS_CLOUD_API_PORT = '13333' # User needs to add their GivEnergy API details GIVENERGY_API_KEY = 'user_givenergy_api_key' +# User needs to add their GivEnergy API details +SOLARMAN_API_URL = 'https://home.solarmanpv.com/maintain-s/history/power' +SOLARMAN_TOKEN = 'user_solarman_token' +SOLARMAN_ID = "user_solarman_id" + # This section is for OpenMeteo setup # Docker is used to fetch and store OpenMeteo's open data, targeting temperature_2m, precipitation, diff --git a/dashboards/dashboard_2/app.py b/dashboards/dashboard_2/app.py index dd0fbf80..80daff37 100644 --- a/dashboards/dashboard_2/app.py +++ b/dashboards/dashboard_2/app.py @@ -21,6 +21,7 @@ from quartz_solar_forecast.inverters.enphase import process_enphase_data from quartz_solar_forecast.inverters.solis import get_solis_data from quartz_solar_forecast.inverters.givenergy import get_givenergy_data +from quartz_solar_forecast.inverters.solarman import get_solarman_data # Load environment variables load_dotenv() @@ -148,7 +149,8 @@ def make_pv_data( access_token: str = None, enphase_system_id: str = None, solis_data: pd.DataFrame = None, - givenergy_data: pd.DataFrame = None + givenergy_data: pd.DataFrame = None, + solarman_data: pd.DataFrame = None ) -> xr.Dataset: live_generation_kw = None @@ -158,6 +160,8 @@ def make_pv_data( live_generation_kw = solis_data elif site.inverter_type == "givenergy" and givenergy_data is not None: live_generation_kw = givenergy_data + elif site.inverter_type == "solarman" and solarman_data is not None: + live_generation_kw = solarman_data da = process_pv_data(live_generation_kw, ts, site) return da @@ -170,7 +174,8 @@ def predict_ocf( access_token: str = None, enphase_system_id: str = None, solis_data: pd.DataFrame = None, - givenergy_data: pd.DataFrame = None + givenergy_data: pd.DataFrame = None, + solarman_data: pd.DataFrame = None ): if ts is None: ts = pd.Timestamp.now().round("15min") @@ -180,7 +185,7 @@ def predict_ocf( nwp_xr = get_nwp(site=site, ts=ts, nwp_source=nwp_source) pv_xr = make_pv_data( site=site, ts=ts, access_token=access_token, enphase_system_id=enphase_system_id, - solis_data=solis_data, givenergy_data=givenergy_data + solis_data=solis_data, givenergy_data=givenergy_data, solarman_data=solarman_data ) pred_df = forecast_v1_tilt_orientation(nwp_source, nwp_xr, pv_xr, ts, model=model) @@ -194,10 +199,11 @@ def run_forecast( access_token: str = None, enphase_system_id: str = None, solis_data: pd.DataFrame = None, - givenergy_data: pd.DataFrame = None + givenergy_data: pd.DataFrame = None, + solarman_data: pd.DataFrame = None ) -> pd.DataFrame: if model == "gb": - return predict_ocf(site, None, ts, nwp_source, access_token, enphase_system_id, solis_data, givenergy_data) + return predict_ocf(site, None, ts, nwp_source, access_token, enphase_system_id, solis_data, givenergy_data, solarman_data) elif model == "xgb": return predict_tryolabs(site, ts) else: @@ -208,7 +214,8 @@ def fetch_data_and_run_forecast( access_token: str = None, enphase_system_id: str = None, solis_data: pd.DataFrame = None, - givenergy_data: pd.DataFrame = None + givenergy_data: pd.DataFrame = None, + solarman_data: pd.DataFrame = None ): with st.spinner("Running forecast..."): try: @@ -225,7 +232,8 @@ def fetch_data_and_run_forecast( access_token=access_token, enphase_system_id=enphase_system_id, solis_data=solis_data, - givenergy_data=givenergy_data + givenergy_data=givenergy_data, + solarman_data=solarman_data ) # Create a site without inverter for comparison @@ -264,12 +272,13 @@ def fetch_data_and_run_forecast( longitude = st.sidebar.number_input("Longitude", min_value=-180.0, max_value=180.0, value=-1.25, step=0.01) capacity_kwp = st.sidebar.number_input("Capacity (kWp)", min_value=0.1, value=1.25, step=0.01) -inverter_type = st.sidebar.selectbox("Select Inverter", ["No Inverter", "Enphase", "Solis", "GivEnergy"]) +inverter_type = st.sidebar.selectbox("Select Inverter", ["No Inverter", "Enphase", "Solis", "GivEnergy", "Solarman"]) access_token = None enphase_system_id = None solis_data = None givenergy_data = None +solarman_data = None if inverter_type == "Enphase": if "access_token" not in st.session_state: @@ -293,6 +302,10 @@ def fetch_data_and_run_forecast( inverter_type=inverter_type.lower() ) + # Define start_date and end_date for Solarman data + start_date = datetime.now() - timedelta(days=7) + end_date = datetime.now() + # Fetch data based on the selected inverter type if inverter_type == "Enphase": predictions_df, ts = fetch_data_and_run_forecast( @@ -308,6 +321,11 @@ def fetch_data_and_run_forecast( predictions_df, ts = fetch_data_and_run_forecast( site, givenergy_data=givenergy_df ) + elif inverter_type == "Solarman": + solarman_df = get_solarman_data(start_date=start_date, end_date=end_date) + predictions_df, ts = fetch_data_and_run_forecast( + site, solarman_data=solarman_df + ) else: predictions_df, ts = fetch_data_and_run_forecast(site) @@ -368,7 +386,6 @@ def fetch_data_and_run_forecast( st.dataframe(predictions_df_display, use_container_width=True) # Some information about the app - st.sidebar.info( """ This dashboard runs @@ -382,4 +399,4 @@ def fetch_data_and_run_forecast( # Footer st.markdown("---") -st.markdown(f"Created with ❤️ by [Open Climate Fix](https://openclimatefix.org/)") +st.markdown(f"Created with ❤️ by [Open Climate Fix](https://openclimatefix.org/)") \ No newline at end of file diff --git a/examples/inverter_example.py b/examples/inverter_example.py index 21c0dd1b..a2a38a3f 100644 --- a/examples/inverter_example.py +++ b/examples/inverter_example.py @@ -13,8 +13,8 @@ def main(save_outputs: bool = False): timestamp_str = datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S') ts = pd.to_datetime(timestamp_str) - # make input data with live enphase, solis, or givenergy data - site_live = PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=1.25, inverter_type="givenergy") # inverter_type="enphase", "solis", or "givenergy" + # make input data with live enphase, solis, givenergy, or solarman data + site_live = PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=1.25, inverter_type="solarman") # inverter_type="enphase", "solis", "givenergy", or "solarman" # make input data with nan data site_no_live = PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=1.25) diff --git a/quartz_solar_forecast/data.py b/quartz_solar_forecast/data.py index 38b8b1d3..cc24c1ca 100644 --- a/quartz_solar_forecast/data.py +++ b/quartz_solar_forecast/data.py @@ -1,6 +1,6 @@ """ Function to get NWP data and create fake PV dataset""" import ssl -from datetime import datetime +from datetime import datetime, timedelta import os import numpy as np import pandas as pd @@ -16,6 +16,7 @@ from quartz_solar_forecast.inverters.enphase import get_enphase_data from quartz_solar_forecast.inverters.solis import get_solis_data from quartz_solar_forecast.inverters.givenergy import get_givenergy_data +from quartz_solar_forecast.inverters.solarman import get_solarman_data ssl._create_default_https_context = ssl._create_unverified_context @@ -153,13 +154,13 @@ def process_pv_data(live_generation_kw: Optional[pd.DataFrame], ts: pd.Timestamp :return: xarray Dataset containing processed PV data """ if live_generation_kw is not None and not live_generation_kw.empty: - # get the most recent data + # Get the most recent data recent_pv_data = live_generation_kw[live_generation_kw['timestamp'] <= ts] - power_kw = np.array([np.array(recent_pv_data["power_kw"].values, dtype=np.float64)]) + power_kw = np.array([recent_pv_data["power_kw"].values], dtype=np.float64) timestamp = recent_pv_data['timestamp'].values else: - # make fake pv data, this is where we could add history of a pv system - power_kw = [[np.nan]] + # Make fake PV data; this is where we could add the history of a PV system + power_kw = np.array([[np.nan]]) timestamp = [ts] da = xr.DataArray( @@ -181,16 +182,14 @@ def process_pv_data(live_generation_kw: Optional[pd.DataFrame], ts: pd.Timestamp def make_pv_data(site: PVSite, ts: pd.Timestamp) -> xr.Dataset: """ - Make PV data by combining live data from Enphase or Solis and fake PV data. - Later we could add PV history here. + Make PV data by combining live data from various inverters. + :param site: the PV site :param ts: the timestamp of the site :return: The combined PV dataset in xarray form """ - # Initialize live_generation_kw to None - live_generation_kw = None + live_generation_kw = None - # Check if the site has an inverter type specified if site.inverter_type == 'enphase': system_id = os.getenv('ENPHASE_SYSTEM_ID') if system_id: @@ -206,9 +205,26 @@ def make_pv_data(site: PVSite, ts: pd.Timestamp) -> xr.Dataset: live_generation_kw = get_givenergy_data() except Exception as e: print(f"Error retrieving GivEnergy data: {str(e)}") + elif site.inverter_type == 'solarman': + try: + end_date = datetime.now() + start_date = end_date - timedelta(weeks=1) + solarman_data = get_solarman_data(start_date, end_date) + + # Filter out rows with null power_kw values + valid_data = solarman_data.dropna(subset=['power_kw']) + + if not valid_data.empty: + # Use all valid data points + live_generation_kw = valid_data + else: + print("No valid Solarman data found.") + live_generation_kw = pd.DataFrame(columns=['timestamp', 'power_kw']) + except Exception as e: + print(f"Error retrieving Solarman data: {str(e)}") + live_generation_kw = pd.DataFrame(columns=['timestamp', 'power_kw']) else: - # If no inverter type is specified or not recognized, set live_generation_kw to None - live_generation_kw = None + live_generation_kw = pd.DataFrame(columns=['timestamp', 'power_kw']) # Process the PV data da = process_pv_data(live_generation_kw, ts, site) diff --git a/quartz_solar_forecast/inverters/solarman.py b/quartz_solar_forecast/inverters/solarman.py new file mode 100644 index 00000000..5b4ce491 --- /dev/null +++ b/quartz_solar_forecast/inverters/solarman.py @@ -0,0 +1,79 @@ +import os +import requests +import pandas as pd +from datetime import timedelta +from dotenv import load_dotenv + +# Load environment variables +load_dotenv() + +# Constants +SOLARMAN_API_URL = os.getenv('SOLARMAN_API_URL') +SOLARMAN_TOKEN = os.getenv('SOLARMAN_TOKEN') +SOLARMAN_ID = os.getenv('SOLARMAN_ID') + +def get_solarman_data(start_date, end_date): + """ + Fetch data from the Solarman API from start_date to end_date. + + :param start_date: Start date (datetime object) + :param end_date: End date (datetime object) + :return: DataFrame with timestamp and power_kw columns + """ + all_data = [] + + current_date = start_date + + while current_date <= end_date: + year = current_date.year + month = current_date.month + day = current_date.day + + url = f"{SOLARMAN_API_URL}/{SOLARMAN_ID}/record" + + headers = { + 'Authorization': f'Bearer {SOLARMAN_TOKEN}', + 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7' + } + + params = { + 'year': year, + 'month': month, + 'day': day + } + + response = requests.get(url, headers=headers, params=params) + + if response.status_code != 200: + print(f"API request failed for {current_date} with status code {response.status_code}") + else: + data = response.json() + records = data.get('records', []) + + if records: + df = pd.DataFrame(records) + df['timestamp'] = pd.to_datetime(df['dateTime'], unit='s') + all_data.append(df) + + current_date += timedelta(days=1) + + if not all_data: + raise ValueError("No data found for the specified date range") + + # Concatenate all dataframes + full_data = pd.concat(all_data, ignore_index=True) + + # Select relevant columns + full_data = full_data[['timestamp', 'generationPower']] + + # Convert watts to kilowatts and rename the column + full_data['power_kw'] = full_data['generationPower'] / 1000.0 + full_data = full_data.drop('generationPower', axis=1) + + # Ensure the dataframe only has 'timestamp' and 'power_kw' columns + full_data = full_data[['timestamp', 'power_kw']] + + # Sort by timestamp + full_data = full_data.sort_values('timestamp') + + return full_data diff --git a/quartz_solar_forecast/pydantic_models.py b/quartz_solar_forecast/pydantic_models.py index 0861b43c..e39a32d1 100644 --- a/quartz_solar_forecast/pydantic_models.py +++ b/quartz_solar_forecast/pydantic_models.py @@ -22,5 +22,5 @@ class PVSite(BaseModel): inverter_type: str = Field( default=None, description="The type of inverter used", - json_schema_extra=["enphase", "solis", "givenergy", None], + json_schema_extra=["enphase", "solis", "givenergy", "solarman", None], )