Weather sync fix maybe

freezing/sync/data/athlete.py

@@ -1,17 +1,12 @@
 from datetime import datetime
 
 from freezing.model import meta
-from freezing.model.orm import (
-    Athlete,
-    Team,
-)
+from freezing.model.orm import Athlete, Team
 from stravalib import model as sm
 
 from freezing.sync.config import config
-from freezing.sync.exc import (
-    MultipleTeamsError,
-    NoTeamsError,
-)
+from freezing.sync.exc import MultipleTeamsError, NoTeamsError
+
 from . import BaseSync, StravaClientForAthlete
 
 
@@ -44,9 +39,13 @@ def sync_athletes(self, max_records: int = None):
                     if not self.all_done():
                         self.register_athlete_team(strava_athlete, athlete)
                 except NoTeamsError as ex:
-                    self.logger.info(f'Athlete "{athlete}" is not on a registered team: {ex}')
+                    self.logger.info(
+                        f'Athlete "{athlete}" is not on a registered team: {ex}'
+                    )
                 except MultipleTeamsError as ex:
-                    self.logger.info(f'Athlete "{athlete}" is on multiple competition teams: {ex}')
+                    self.logger.info(
+                        f'Athlete "{athlete}" is on multiple competition teams: {ex}'
+                    )
                 except Exception:
                     self.logger.exception(
                         "Error registering athlete {0}".format(athlete), exc_info=True

freezing/sync/data/weather.py

@@ -1,9 +1,10 @@
 import logging
-from datetime import timedelta
+from datetime import datetime, timedelta
 from decimal import Decimal
 from statistics import mean
 
 from freezing.model import meta, orm
+from pytz import timezone
 from sqlalchemy import text
 
 from freezing.sync.config import config
@@ -45,14 +46,15 @@ def sync_weather(
         sess.query(orm.RideWeather)
         q = text(
             """
-            select R.id from rides R
+            select R.id, ST_AsText(G.start_geo) AS start_geo from rides R
             join ride_geo G on G.ride_id = R.id
             left join ride_weather W on W.ride_id = R.id
             where W.ride_id is null
-            and date(R.start_date) < CURDATE() -- Only include rides from yesterday or before
-            and time(R.start_date) != '00:00:00' -- Exclude bad entries.
+            and date_add(CONVERT_TZ(R.start_date, R.timezone, '{0}'), INTERVAL R.elapsed_time SECOND) < (NOW() - INTERVAL 1 HOUR) -- Only include rides that ended over an hour ago
             ;
-            """
+            """.format(
+                config.TIMEZONE
+            )
         )
 
         visual_crossing = HistoVisualCrossing(
@@ -71,15 +73,18 @@ def sync_weather(
                 break
 
             ride = sess.query(orm.Ride).get(r["id"])
+            start_geo_wkt = r["start_geo"]
             self.logger.info(
-                "Processing ride: {0} ({1}/{2})".format(ride.id, i, num_rides)
+                "Processing ride: {0} ({1}/{2}) ({3})".format(
+                    ride.id, i, num_rides, start_geo_wkt
+                )
             )
 
             try:
                 # If you can't reproduce the ancient infrastructure required by all this and so can't run any of the
                 # geoalchemy stuff you can hardcode this to debug
                 # start_geo_wkt = "POINT(-76.96 38.96)"
-                start_geo_wkt = meta.scoped_session().scalar(ride.geo.start_geo.wkt)
+                # start_geo_wkt = meta.scoped_session().scalar(ride.geo.start_geo.wkt)
                 point = parse_point_wkt(start_geo_wkt)
 
                 # We round lat/lon to decrease the granularity and allow better re-use of cache data.
@@ -93,13 +98,31 @@ def sync_weather(
                     )
                 )
 
+                ride_today = datetime.now(timezone(ride.timezone))
+                start_date = ride.start_date.replace(tzinfo=timezone(ride.timezone))
+                fetch_date = start_date + timedelta(seconds=ride.elapsed_time)
+                # For caching purposes we're saying we want weather as of the end of the ride, so if
+                # we have weather from earlier in the day we don't use it. Because we're lame and
+                # don't want to handle rides that span midnight, we max to 23:59 of the day. If
+                # we are fetching old weather, we also just ask for the end of the day so we will
+                # use the latest cache file.
+                if (
+                    fetch_date.date() < ride_today.date()
+                    or fetch_date.date() != start_date.date()
+                ):
+                    fetch_date = start_date.replace(
+                        hour=23, minute=59, second=0, microsecond=0
+                    )
+
+                # VC gives us back weather in the timezone of the lat/lon that we asked. So we ask for
+                # weather in the ride-local date and interpret times accordingly.
                 hist = visual_crossing.histo_forecast(
-                    time=ride.start_date, latitude=lat, longitude=lon
+                    time=fetch_date, latitude=lat, longitude=lon
                 )
 
                 self.logger.debug("Got response in timezone {0}".format(hist.timezone))
 
-                ride_start = ride.start_date.replace(tzinfo=hist.timezone)
+                ride_start = start_date.astimezone(tz=hist.timezone)
                 ride_end = ride_start + timedelta(seconds=ride.elapsed_time)
 
                 # NOTE: if elapsed_time is significantly more than moving_time then we need to assume

freezing/sync/run.py

@@ -46,10 +46,10 @@ def segmented_sync_activities():
     # This should generally not pick up anytihng.
     scheduler.add_job(activity_sync.sync_rides_detail, "cron", minute="20")
 
-    # Sync weather at 8am UTC
-    scheduler.add_job(weather_sync.sync_weather, "cron", hour="8")
+    # Sync weather every hour
+    scheduler.add_job(weather_sync.sync_weather, "cron", minute="45")
 
-    # Sync athletes once a day at 6am UTC
+    # Sync athletes every hour
     scheduler.add_job(athlete_sync.sync_athletes, "cron", minute="30")
 
     scheduler.start()

freezing/sync/wx/visualcrossing/api.py

@@ -61,7 +61,7 @@ def _cache_file(self, time: datetime, longitude: float, latitude: float):
         if not self.cache_dir:
             return None  # where are all the monads
         directory = os.path.join(self.cache_dir, f"{longitude}x{latitude}")
-        return os.path.join(directory, f'{time.strftime("%Y-%m-%d")}.json')
+        return os.path.join(directory, f'{time.strftime("%Y-%m-%dT%H")}.json')
 
     def _get_cached(self, path: str, fetch):
         if not path: