ran both providers locally

pvconsumer/app.py

@@ -128,6 +128,7 @@ def pull_data_and_save(
     pv_system_chunks = chunks(original_list=pv_systems, n=n_pv_systems_per_batch)
 
     all_pv_yields_df = []
+    i = 0
     for pv_system_chunk in pv_system_chunks:
         if provider == "pvoutput.org":
             # set up pv output.org
@@ -155,7 +156,8 @@ def pull_data_and_save(
         else:
             raise Exception(f"Can not use provider {provider}")
 
-        for i, pv_system in enumerate(pv_system_chunk):
+        for _, pv_system in enumerate(pv_system_chunk):
+            i = i + 1
             logger.debug(
                 f"Processing {i}th pv system ({pv_system.client_site_id=}), "
                 f"out of {len(pv_systems)}"

pvconsumer/pv_systems.py

@@ -144,6 +144,7 @@ def get_pv_systems(
                 pv_system.latitude = metadata.latitude
                 pv_system.longitude = metadata.longitude
                 pv_system.status_interval_minutes = int(metadata.status_interval_minutes)
+                pv_system.capacity_kw = metadata.system_DC_capacity_W / 1000
 
             elif provider == solar_sheffield_passiv:
                 pv_system = pv_systems[pv_systems["pv_system_id"] == pv_system.pv_system_id].iloc[0]
@@ -214,11 +215,12 @@ def filter_pv_systems_which_have_new_data(
         session.query(SiteSQL.site_uuid, GenerationSQL.start_utc)
         .distinct(
             GenerationSQL.site_uuid,
-            GenerationSQL.start_utc,
+            # GenerationSQL.start_utc,
         )
         .join(SiteSQL)
         .filter(
             GenerationSQL.start_utc <= datetime_utc,
+            GenerationSQL.start_utc >= datetime_utc - timedelta(days=1),
             GenerationSQL.site_uuid.in_(site_uuids),
         )
         .order_by(

pvconsumer/utils.py

@@ -1,6 +1,6 @@
 """ Utils functions """
 import logging
-from datetime import timezone
+from datetime import datetime, timedelta, timezone
 
 import pandas as pd
 from pvsite_datamodel.sqlmodels import GenerationSQL, SiteSQL
@@ -51,10 +51,17 @@ def format_pv_data(pv_system: SiteSQL, pv_yield_df: pd.DataFrame, session: Sessi
             pv_yield_df.drop(pv_yield_df.tail(1).index, inplace=True)
 
     # 2. filter by last
+    if len(pv_yield_df) > 0:
+        start_utc_filter = pv_yield_df["datetime_utc"].min() - timedelta(days=1)
+    else:
+        start_utc_filter = datetime.now() - timedelta(days=1)
+
     last_pv_generation = (
         session.query(GenerationSQL)
+        .filter(GenerationSQL.site_uuid == pv_system.site_uuid)
         .join(SiteSQL)
         .filter(SiteSQL.site_uuid == pv_system.site_uuid)
+        .filter(GenerationSQL.start_utc > start_utc_filter)
         .order_by(GenerationSQL.created_utc.desc())
         .first()
     )