Skip to content

Commit

Permalink
Do not request data from std sites (8)
Browse files Browse the repository at this point in the history
Sort queries and improve prints
  • Loading branch information
dostuffthatmatters committed Dec 26, 2023
1 parent 535f545 commit 0da7278
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 5 deletions.
5 changes: 3 additions & 2 deletions src/profiles/generate_queries.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,8 @@ def remove_std_site_data(
set[datetime.date]] = copy.deepcopy(missing_data)
for std_site_config in config.profiles.GGG2020_standard_sites:
location = ProfilesQueryLocation(
lat=std_site_config.lat, lon=std_site_config.lon
lat=round(std_site_config.lat),
lon=round(std_site_config.lon),
)
if location in filtered_data.keys():
filtered_data[location].difference_update(
Expand Down Expand Up @@ -257,4 +258,4 @@ def generate_download_queries(
) for tp in compute_time_periods(dates)
])

return download_queries
return sorted(download_queries, key=lambda q: q.from_date)
7 changes: 4 additions & 3 deletions src/profiles/upload_logic.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ def upload_requests(
# job every minute, so we need to wait for the cron job to
# finish before we can upload the next request
request_time = (t2 - t1)
if request_time < 65:
progress.print(f"Sleeping for {65 - request_time} seconds")
time.sleep(65 - request_time)
sleep_time = max(round(65 - request_time, 3), 0)
if sleep_time > 0:
progress.print(f"Sleeping for {sleep_time} seconds")
time.sleep(sleep_time)

0 comments on commit 0da7278

Please sign in to comment.