diff --git a/docs/source/usage.md b/docs/source/usage.md index 6a665e40..ea4d7911 100644 --- a/docs/source/usage.md +++ b/docs/source/usage.md @@ -292,6 +292,8 @@ The full set of configuration options are: (Default: `https://www.googleapis.com/auth/gmail.modify`) - `oauth2_port` - int: The TCP port for the local server to listen on for the OAuth2 response (Default: `8080`) + - `paginate_messages` - bool: When `True`, fetch all applicable Gmail messages. + When `False`, only fetch up to 100 new messages per run (Default: `True`) - `log_analytics` - `client_id` - str: The app registration's client ID - `client_secret` - str: The app registration's client secret diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 445b3e40..4ef33f84 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -335,6 +335,7 @@ def process_reports(reports_): gmail_api_credentials_file=None, gmail_api_token_file=None, gmail_api_include_spam_trash=False, + gmail_api_paginate_messages=True, gmail_api_scopes=[], gmail_api_oauth2_port=8080, log_file=args.log_file, @@ -750,6 +751,8 @@ def process_reports(reports_): gmail_api_config.get("token_file", ".token") opts.gmail_api_include_spam_trash = \ gmail_api_config.getboolean("include_spam_trash", False) + opts.gmail_api_paginate_messages = \ + gmail_api_config.getboolean("paginate_messages", True) opts.gmail_api_scopes = \ gmail_api_config.get("scopes", default_gmail_api_scope) @@ -1009,6 +1012,7 @@ def process_reports(reports_): token_file=opts.gmail_api_token_file, scopes=opts.gmail_api_scopes, include_spam_trash=opts.gmail_api_include_spam_trash, + paginate_messages=opts.gmail_api_paginate_messages, reports_folder=opts.mailbox_reports_folder, oauth2_port=opts.gmail_api_oauth2_port ) diff --git a/parsedmarc/mail/gmail.py b/parsedmarc/mail/gmail.py index 997fda20..436e1f02 100644 --- a/parsedmarc/mail/gmail.py +++ b/parsedmarc/mail/gmail.py @@ -42,11 +42,13 @@ def __init__(self, scopes: List[str], include_spam_trash: bool, reports_folder: str, - oauth2_port: int): + oauth2_port: int, + paginate_messages: bool): creds = _get_creds(token_file, credentials_file, scopes, oauth2_port) self.service = build('gmail', 'v1', credentials=creds) self.include_spam_trash = include_spam_trash self.reports_label_id = self._find_label_id_for_label(reports_folder) + self.paginate_messages = paginate_messages def create_folder(self, folder_name: str): # Gmail doesn't support the name Archive @@ -81,7 +83,7 @@ def _fetch_all_message_ids(self, reports_label_id, page_token=None): for message in messages: yield message["id"] - if "nextPageToken" in results: + if "nextPageToken" in results and self.paginate_messages: yield from self._fetch_all_message_ids( reports_label_id, results["nextPageToken"] )