solarman (#172)

* solarman * solarman * renamed col * na data rm
openclimatefix · Aug 6, 2024 · b403b6a · b403b6a
1 parent ec0983a
commit b403b6a
Show file tree

Hide file tree

Showing 6 changed files with 142 additions and 25 deletions.
diff --git a/.env.example b/.env.example
@@ -15,6 +15,11 @@ SOLIS_CLOUD_API_PORT = '13333'
 # User needs to add their GivEnergy API details
 GIVENERGY_API_KEY = 'user_givenergy_api_key'
 
+# User needs to add their GivEnergy API details
+SOLARMAN_API_URL = 'https://home.solarmanpv.com/maintain-s/history/power'
+SOLARMAN_TOKEN = 'user_solarman_token'
+SOLARMAN_ID = "user_solarman_id"
+
 # This section is for OpenMeteo setup
 
 # Docker is used to fetch and store OpenMeteo's open data, targeting temperature_2m, precipitation, 

diff --git a/dashboards/dashboard_2/app.py b/dashboards/dashboard_2/app.py
@@ -21,6 +21,7 @@
 from quartz_solar_forecast.inverters.enphase import process_enphase_data
 from quartz_solar_forecast.inverters.solis import get_solis_data
 from quartz_solar_forecast.inverters.givenergy import get_givenergy_data
+from quartz_solar_forecast.inverters.solarman import get_solarman_data 
 
 # Load environment variables
 load_dotenv()
@@ -148,7 +149,8 @@ def make_pv_data(
     access_token: str = None,
     enphase_system_id: str = None,
     solis_data: pd.DataFrame = None,
-    givenergy_data: pd.DataFrame = None
+    givenergy_data: pd.DataFrame = None,
+    solarman_data: pd.DataFrame = None
 ) -> xr.Dataset:
     live_generation_kw = None
 
@@ -158,6 +160,8 @@ def make_pv_data(
         live_generation_kw = solis_data
     elif site.inverter_type == "givenergy" and givenergy_data is not None:
         live_generation_kw = givenergy_data
+    elif site.inverter_type == "solarman" and solarman_data is not None:
+        live_generation_kw = solarman_data
 
     da = process_pv_data(live_generation_kw, ts, site)
     return da
@@ -170,7 +174,8 @@ def predict_ocf(
     access_token: str = None,
     enphase_system_id: str = None,
     solis_data: pd.DataFrame = None,
-    givenergy_data: pd.DataFrame = None
+    givenergy_data: pd.DataFrame = None,
+    solarman_data: pd.DataFrame = None
 ):
     if ts is None:
         ts = pd.Timestamp.now().round("15min")
@@ -180,7 +185,7 @@ def predict_ocf(
     nwp_xr = get_nwp(site=site, ts=ts, nwp_source=nwp_source)
     pv_xr = make_pv_data(
         site=site, ts=ts, access_token=access_token, enphase_system_id=enphase_system_id, 
-        solis_data=solis_data, givenergy_data=givenergy_data
+        solis_data=solis_data, givenergy_data=givenergy_data, solarman_data=solarman_data
     )
 
     pred_df = forecast_v1_tilt_orientation(nwp_source, nwp_xr, pv_xr, ts, model=model)
@@ -194,10 +199,11 @@ def run_forecast(
     access_token: str = None,
     enphase_system_id: str = None,
     solis_data: pd.DataFrame = None,
-    givenergy_data: pd.DataFrame = None
+    givenergy_data: pd.DataFrame = None,
+    solarman_data: pd.DataFrame = None
 ) -> pd.DataFrame:
     if model == "gb":
-        return predict_ocf(site, None, ts, nwp_source, access_token, enphase_system_id, solis_data, givenergy_data)
+        return predict_ocf(site, None, ts, nwp_source, access_token, enphase_system_id, solis_data, givenergy_data, solarman_data)
     elif model == "xgb":
         return predict_tryolabs(site, ts)
     else:
@@ -208,7 +214,8 @@ def fetch_data_and_run_forecast(
     access_token: str = None,
     enphase_system_id: str = None,
     solis_data: pd.DataFrame = None,
-    givenergy_data: pd.DataFrame = None
+    givenergy_data: pd.DataFrame = None,
+    solarman_data: pd.DataFrame = None
 ):
     with st.spinner("Running forecast..."):
         try:
@@ -225,7 +232,8 @@ def fetch_data_and_run_forecast(
                 access_token=access_token,
                 enphase_system_id=enphase_system_id,
                 solis_data=solis_data,
-                givenergy_data=givenergy_data
+                givenergy_data=givenergy_data,
+                solarman_data=solarman_data
             )
 
             # Create a site without inverter for comparison
@@ -264,12 +272,13 @@ def fetch_data_and_run_forecast(
     longitude = st.sidebar.number_input("Longitude", min_value=-180.0, max_value=180.0, value=-1.25, step=0.01)
     capacity_kwp = st.sidebar.number_input("Capacity (kWp)", min_value=0.1, value=1.25, step=0.01)
 
-inverter_type = st.sidebar.selectbox("Select Inverter", ["No Inverter", "Enphase", "Solis", "GivEnergy"])
+inverter_type = st.sidebar.selectbox("Select Inverter", ["No Inverter", "Enphase", "Solis", "GivEnergy", "Solarman"])
 
 access_token = None
 enphase_system_id = None
 solis_data = None
 givenergy_data = None
+solarman_data = None
 
 if inverter_type == "Enphase":
     if "access_token" not in st.session_state:
@@ -293,6 +302,10 @@ def fetch_data_and_run_forecast(
             inverter_type=inverter_type.lower()
         )
 
+        # Define start_date and end_date for Solarman data
+        start_date = datetime.now() - timedelta(days=7)
+        end_date = datetime.now()
+
         # Fetch data based on the selected inverter type
         if inverter_type == "Enphase":
             predictions_df, ts = fetch_data_and_run_forecast(
@@ -308,6 +321,11 @@ def fetch_data_and_run_forecast(
             predictions_df, ts = fetch_data_and_run_forecast(
                 site, givenergy_data=givenergy_df
             )
+        elif inverter_type == "Solarman":
+            solarman_df = get_solarman_data(start_date=start_date, end_date=end_date)
+            predictions_df, ts = fetch_data_and_run_forecast(
+                site, solarman_data=solarman_df
+            )
         else:
             predictions_df, ts = fetch_data_and_run_forecast(site)
 
@@ -368,7 +386,6 @@ def fetch_data_and_run_forecast(
             st.dataframe(predictions_df_display, use_container_width=True)
 
 # Some information about the app
-
 st.sidebar.info(
     """
     This dashboard runs
@@ -382,4 +399,4 @@ def fetch_data_and_run_forecast(
 
 # Footer
 st.markdown("---")
-st.markdown(f"Created with ❤️ by [Open Climate Fix](https://openclimatefix.org/)")
+st.markdown(f"Created with ❤️ by [Open Climate Fix](https://openclimatefix.org/)")
diff --git a/examples/inverter_example.py b/examples/inverter_example.py
@@ -13,8 +13,8 @@ def main(save_outputs: bool = False):
     timestamp_str = datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
     ts = pd.to_datetime(timestamp_str)
 
-    # make input data with live enphase, solis, or givenergy data
-    site_live = PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=1.25, inverter_type="givenergy") # inverter_type="enphase", "solis", or "givenergy"
+    # make input data with live enphase, solis, givenergy, or solarman data
+    site_live = PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=1.25, inverter_type="solarman") # inverter_type="enphase", "solis", "givenergy", or "solarman"
 
     # make input data with nan data
     site_no_live = PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=1.25)

diff --git a/quartz_solar_forecast/data.py b/quartz_solar_forecast/data.py
@@ -1,6 +1,6 @@
 """ Function to get NWP data and create fake PV dataset"""
 import ssl
-from datetime import datetime
+from datetime import datetime, timedelta
 import os  
 import numpy as np
 import pandas as pd
@@ -16,6 +16,7 @@
 from quartz_solar_forecast.inverters.enphase import get_enphase_data
 from quartz_solar_forecast.inverters.solis import get_solis_data
 from quartz_solar_forecast.inverters.givenergy import get_givenergy_data
+from quartz_solar_forecast.inverters.solarman import get_solarman_data
 
 ssl._create_default_https_context = ssl._create_unverified_context
 
@@ -153,13 +154,13 @@ def process_pv_data(live_generation_kw: Optional[pd.DataFrame], ts: pd.Timestamp
     :return: xarray Dataset containing processed PV data
     """
     if live_generation_kw is not None and not live_generation_kw.empty:
-        # get the most recent data
+        # Get the most recent data
         recent_pv_data = live_generation_kw[live_generation_kw['timestamp'] <= ts]
-        power_kw = np.array([np.array(recent_pv_data["power_kw"].values, dtype=np.float64)])
+        power_kw = np.array([recent_pv_data["power_kw"].values], dtype=np.float64)
         timestamp = recent_pv_data['timestamp'].values
     else:
-        # make fake pv data, this is where we could add history of a pv system
-        power_kw = [[np.nan]]
+        # Make fake PV data; this is where we could add the history of a PV system
+        power_kw = np.array([[np.nan]])
         timestamp = [ts]
 
     da = xr.DataArray(
@@ -181,16 +182,14 @@ def process_pv_data(live_generation_kw: Optional[pd.DataFrame], ts: pd.Timestamp
 
 def make_pv_data(site: PVSite, ts: pd.Timestamp) -> xr.Dataset:
     """
-    Make PV data by combining live data from Enphase or Solis and fake PV data.
-    Later we could add PV history here.
+    Make PV data by combining live data from various inverters.
+    
     :param site: the PV site
     :param ts: the timestamp of the site
     :return: The combined PV dataset in xarray form
     """
-    # Initialize live_generation_kw to None
-    live_generation_kw = None  
+    live_generation_kw = None
 
-    # Check if the site has an inverter type specified
     if site.inverter_type == 'enphase':
         system_id = os.getenv('ENPHASE_SYSTEM_ID')
         if system_id:
@@ -206,9 +205,26 @@ def make_pv_data(site: PVSite, ts: pd.Timestamp) -> xr.Dataset:
             live_generation_kw = get_givenergy_data()
         except Exception as e:
             print(f"Error retrieving GivEnergy data: {str(e)}")
+    elif site.inverter_type == 'solarman':
+        try:
+            end_date = datetime.now()
+            start_date = end_date - timedelta(weeks=1)
+            solarman_data = get_solarman_data(start_date, end_date)
+
+            # Filter out rows with null power_kw values
+            valid_data = solarman_data.dropna(subset=['power_kw'])
+
+            if not valid_data.empty:
+                # Use all valid data points
+                live_generation_kw = valid_data
+            else:
+                print("No valid Solarman data found.")
+                live_generation_kw = pd.DataFrame(columns=['timestamp', 'power_kw'])
+        except Exception as e:
+            print(f"Error retrieving Solarman data: {str(e)}")
+            live_generation_kw = pd.DataFrame(columns=['timestamp', 'power_kw'])
     else:
-        # If no inverter type is specified or not recognized, set live_generation_kw to None
-        live_generation_kw = None
+        live_generation_kw = pd.DataFrame(columns=['timestamp', 'power_kw'])
 
     # Process the PV data
     da = process_pv_data(live_generation_kw, ts, site)

diff --git a/quartz_solar_forecast/inverters/solarman.py b/quartz_solar_forecast/inverters/solarman.py
@@ -0,0 +1,79 @@
+import os
+import requests
+import pandas as pd
+from datetime import timedelta
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+# Constants
+SOLARMAN_API_URL = os.getenv('SOLARMAN_API_URL')
+SOLARMAN_TOKEN = os.getenv('SOLARMAN_TOKEN')
+SOLARMAN_ID = os.getenv('SOLARMAN_ID')
+
+def get_solarman_data(start_date, end_date):
+    """
+    Fetch data from the Solarman API from start_date to end_date.
+    
+    :param start_date: Start date (datetime object)
+    :param end_date: End date (datetime object)
+    :return: DataFrame with timestamp and power_kw columns
+    """
+    all_data = []
+
+    current_date = start_date
+
+    while current_date <= end_date:
+        year = current_date.year
+        month = current_date.month
+        day = current_date.day
+
+        url = f"{SOLARMAN_API_URL}/{SOLARMAN_ID}/record"
+
+        headers = {
+            'Authorization': f'Bearer {SOLARMAN_TOKEN}',
+            'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'
+        }
+
+        params = {
+            'year': year,
+            'month': month,
+            'day': day
+        }
+
+        response = requests.get(url, headers=headers, params=params)
+
+        if response.status_code != 200:
+            print(f"API request failed for {current_date} with status code {response.status_code}")
+        else:
+            data = response.json()
+            records = data.get('records', [])
+
+            if records:
+                df = pd.DataFrame(records)
+                df['timestamp'] = pd.to_datetime(df['dateTime'], unit='s')
+                all_data.append(df)
+
+        current_date += timedelta(days=1)
+
+    if not all_data:
+        raise ValueError("No data found for the specified date range")
+
+    # Concatenate all dataframes
+    full_data = pd.concat(all_data, ignore_index=True)
+
+    # Select relevant columns
+    full_data = full_data[['timestamp', 'generationPower']]
+
+    # Convert watts to kilowatts and rename the column
+    full_data['power_kw'] = full_data['generationPower'] / 1000.0
+    full_data = full_data.drop('generationPower', axis=1)
+
+    # Ensure the dataframe only has 'timestamp' and 'power_kw' columns
+    full_data = full_data[['timestamp', 'power_kw']]
+
+    # Sort by timestamp
+    full_data = full_data.sort_values('timestamp')
+
+    return full_data
diff --git a/quartz_solar_forecast/pydantic_models.py b/quartz_solar_forecast/pydantic_models.py
@@ -22,5 +22,5 @@ class PVSite(BaseModel):
     inverter_type: str = Field(
         default=None,
         description="The type of inverter used",
-        json_schema_extra=["enphase", "solis", "givenergy", None],
+        json_schema_extra=["enphase", "solis", "givenergy", "solarman", None],
     )