diff --git a/tap_restaurant365/streams.py b/tap_restaurant365/streams.py index cabe4e6..9db9c76 100644 --- a/tap_restaurant365/streams.py +++ b/tap_restaurant365/streams.py @@ -32,17 +32,15 @@ def get_next_page_token( previous_token = previous_token['token'] return {"token":previous_token,"skip":self.skip} else: - self.skip = 0 - start_date = (parser.parse(self.tap_state["bookmarks"][self.name]['starting_replication_value']) + timedelta(seconds=1)) or parser.parse(self.config.get("start_date")) + self.skip = 0 + #Pick starting value just incase progress marker is not present. + replication_key_value = self.tap_state["bookmarks"][self.name]['starting_replication_value'] + if "progress_markers" in self.tap_state["bookmarks"][self.name]: + replication_key_value = self.tap_state["bookmarks"][self.name]['progress_markers']["replication_key_value"] + + start_date = (parser.parse(replication_key_value) + timedelta(seconds=1)) or parser.parse(self.config.get("start_date")) today = datetime.today() - if previous_token: - if "token" in previous_token: - previous_token = previous_token['token'] - if previous_token: - #Look for records that are greater than the previous token - previous_token = previous_token + timedelta(seconds=1) - previous_token = previous_token or start_date - next_token = (previous_token + timedelta(days=self.days_delta)).replace(tzinfo=None) + next_token = start_date.replace(tzinfo=None) if (today - next_token).days < self.days_delta: self.paginate = False @@ -64,7 +62,10 @@ def get_url_params( start_date = token_date or self.get_starting_time(context) end_date = start_date + timedelta(days = self.days_delta) if self.replication_key: - params["$filter"] = f"{self.replication_key} ge {start_date.strftime('%Y-%m-%dT%H:%M:%SZ')} and {self.replication_key} lt {end_date.strftime('%Y-%m-%dT%H:%M:%SZ')}" + params["$filter"] = ( + f"{self.replication_key} ge {start_date.strftime('%Y-%m-%dT%H:%M:%SZ')} and {self.replication_key} lt {end_date.strftime('%Y-%m-%dT23:59:59Z')}" + ) + params['$orderby'] = f"{self.replication_key}" if self.name == "journal_entries": #set a date in the stream to check later to see if we need to keep calling to the stream params["$filter"] += f" and type eq 'Journal Entry'"