diff --git a/meters/match_field_processing.py b/meters/match_field_processing.py index c3b27b4..e35ac63 100644 --- a/meters/match_field_processing.py +++ b/meters/match_field_processing.py @@ -66,7 +66,7 @@ def get_fiserv(pstgrs): """ params = { "select": "id,invoice_id,transaction_date,flowbird_id", - "order": "id", + "order": "id.desc", "flowbird_id": "is.null", "transaction_date": "not.is.null", "invoice_id": "not.is.null", @@ -97,7 +97,7 @@ def get_payments(pstgrs, start, end): """ params = { "select": "id,invoice_id,transaction_date,updated_at", - "order": "id", + "order": "id.desc", "and": f"(updated_at.lte.{end},updated_at.gte.{start})", } payments = pstgrs.select(resource="flowbird_payments_raw", params=params) diff --git a/meters/parking_socrata.py b/meters/parking_socrata.py index 3047ec5..871c10e 100644 --- a/meters/parking_socrata.py +++ b/meters/parking_socrata.py @@ -102,15 +102,16 @@ def batch_upload(start, end, pstgrs, soda, table): logger.debug(f"Publishing table: {table} to Socrata from {start} to {end}") paginate = True offset = 0 + chunk_size = 1000 while paginate: params = { "select": "*", "and": f"(updated_at.lte.{end},updated_at.gte.{start})", - "order": "id", - "limit": 1000, + "order": "id.desc", + "limit": chunk_size, "offset": offset, } - offset += 1000 + offset += chunk_size response = pstgrs.select(resource=table, params=params, pagination=True) if len(response) == 0: paginate = False @@ -126,7 +127,6 @@ def main(args): pstgrs = Postgrest( POSTGREST_ENDPOINT, token=POSTGREST_TOKEN, - headers={"Prefer": "return=representation"}, ) # sodapy soda = Socrata(SO_WEB, SO_TOKEN, username=SO_USER, password=SO_PASS, timeout=500,)