Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Journals binarysearch offset #14

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion tap_xero/streams.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,9 +179,42 @@ class Journals(Stream):
"""The Journals endpoint is a special case. It has its own way of ordering
and paging the data. See
https://developer.xero.com/documentation/api/journals"""

def get_max_journal(self,ctx, offset=None):

start = 0
end = 100000000
iterations = 1
starting_offset = 0
while start <= end:
print(f"start is {start} and end is {end}")
mid = (start + end) // 2
params = {"offset": mid}
response = _make_request(ctx, self.tap_stream_id, params)
#Check if journals are found in the response
journal_numbers = [entry.get('JournalNumber', 0) for entry in response]
if journal_numbers:
max_journal = max(journal_numbers)
#Adjust to 500k offset
starting_offset = max_journal - int(ctx.config.get("journal_lookback_range",499900))
if starting_offset < 0:
starting_offset = 0
break

#Keep adjusting the offset and the search interval
if end - start < 1000: #@TODO adjust it to desired offset e.g 500,000
break
elif mid < 1000:
start = mid + 1
else:
end = mid - 1
logging.info(f"iteration for calculating Journal offset {iterations} completed")
iterations += 1
return starting_offset

def sync(self, ctx):
bookmark = [self.tap_stream_id, self.bookmark_key]
journal_number = ctx.get_bookmark(bookmark) or 0
journal_number = ctx.get_bookmark(bookmark) or self.get_max_journal(ctx)
while True:
filter_options = {"offset": journal_number}
if ctx.config.get("start_date"):
Expand Down Expand Up @@ -235,6 +268,7 @@ def write_records(self, records, ctx):
line = transformer.transform(line, lines_schema, metadata.to_map(line_mdata))
singer.write_record(lines_stream_id, line)
self.metrics(records)



class LinkedTransactions(Stream):
Expand Down