From 126832fa726d8432cf0d7d2da02337b2a3772da0 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 13:44:10 +0000 Subject: [PATCH 01/34] fix: render commands available in dev only --- app/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/main.py b/app/main.py index 3d2c3c2d..f2565389 100644 --- a/app/main.py +++ b/app/main.py @@ -26,9 +26,10 @@ def main(bot): APP_TOKEN = os.environ.get("APP_TOKEN") PREFIX = os.environ.get("PREFIX", "") - # Register Google Service command - bot.command(f"/{PREFIX}google-service")(google_service.google_service_command) - bot.view("google_service_view")(google_service.open_modal) + # Register Google Service command for dev purposes only + if os.getenv('PREFIX') == 'dev-': + bot.command(f"/{PREFIX}google-service")(google_service.google_service_command) + bot.view("google_service_view")(google_service.open_modal) # Register Roles commands bot.command(f"/{PREFIX}talent-role")(role.role_command) From 3f0aa598d94a66920fbb41373d2ebb224e5d0141 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:02:10 +0000 Subject: [PATCH 02/34] fix: move google-service command to bottom --- app/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/main.py b/app/main.py index f2565389..482c900f 100644 --- a/app/main.py +++ b/app/main.py @@ -26,11 +26,6 @@ def main(bot): APP_TOKEN = os.environ.get("APP_TOKEN") PREFIX = os.environ.get("PREFIX", "") - # Register Google Service command for dev purposes only - if os.getenv('PREFIX') == 'dev-': - bot.command(f"/{PREFIX}google-service")(google_service.google_service_command) - bot.view("google_service_view")(google_service.open_modal) - # Register Roles commands bot.command(f"/{PREFIX}talent-role")(role.role_command) bot.view("role_view")(role.role_view_handler) @@ -99,6 +94,11 @@ def main(bot): stop_run_continuously = scheduled_tasks.run_continuously() server_app.add_event_handler("shutdown", lambda: stop_run_continuously.set()) + # Register Google Service command for dev purposes only + if PREFIX == "dev-": + bot.command(f"/{PREFIX}google-service")(google_service.google_service_command) + bot.view("google_service_view")(google_service.open_modal) + def get_bot(): SLACK_TOKEN = os.environ.get("SLACK_TOKEN", None) From 4355ba863a552ceeb06fd34308d39101eede2864 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:02:42 +0000 Subject: [PATCH 03/34] feat: make test command available in dev only --- app/commands/sre.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/commands/sre.py b/app/commands/sre.py index 355f58bf..7749ae19 100644 --- a/app/commands/sre.py +++ b/app/commands/sre.py @@ -21,6 +21,8 @@ \n - show the version of the SRE Bot \n - montre la version du bot SRE""" +PREFIX = os.environ.get("PREFIX", "") + def sre_command(ack, command, logger, respond, client, body): ack() @@ -48,7 +50,11 @@ def sre_command(ack, command, logger, respond, client, body): case "version": respond(f"SRE Bot version: {os.environ.get('GIT_SHA', 'unknown')}") case "google-service": - google_service.google_service_command(client, body) + if PREFIX == "dev-": + google_service.google_service_command(client, body) + else: + respond("This command is only available in the dev environment.") + return case _: respond( f"Unknown command: `{action}`. Type `/sre help` to see a list of commands. \nCommande inconnue: `{action}`. Entrez `/sre help` pour une liste des commandes valides" From 3cc40c76948c7152fa37629a9d419164a4528775 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:38:16 +0000 Subject: [PATCH 04/34] fix: fix mypy path for current project structure --- .vscode/settings.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 6e4f5b5d..e451174a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,4 @@ { "python.pylintPath": "/usr/local/py-utils/bin/pylint", + "mypy-type-checker.cwd": "${workspaceFolder}/app", } \ No newline at end of file From a1ad43f9c877da33b3ef8f9d569e691a2cf3dc44 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:48:02 +0000 Subject: [PATCH 05/34] fix: update pylint cwd --- .vscode/settings.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index e451174a..3d25dc7a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,5 @@ { "python.pylintPath": "/usr/local/py-utils/bin/pylint", "mypy-type-checker.cwd": "${workspaceFolder}/app", + "pylint.cwd": "${workspaceFolder}/app", } \ No newline at end of file From 060b0bdf288c5e6230ef60d66184fa1165415327 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:48:22 +0000 Subject: [PATCH 06/34] fix: ignore google.oauth2 missing stub warning --- app/mypy.ini | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 app/mypy.ini diff --git a/app/mypy.ini b/app/mypy.ini new file mode 100644 index 00000000..0bb31247 --- /dev/null +++ b/app/mypy.ini @@ -0,0 +1,2 @@ +[mypy-google.oauth2.service_account] +ignore_missing_imports = True \ No newline at end of file From 47742146d4cf63dc8457be16845b0fc1476a1ab1 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 16:00:38 +0000 Subject: [PATCH 07/34] fix: config flake8 instead of pylint --- .devcontainer/devcontainer.json | 1 - .vscode/settings.json | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 88a47015..bda12a20 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,7 +11,6 @@ "ms-python.python", "ms-python.vscode-pylance", "ms-python.flake8", - "ms-python.pylint", "ms-python.mypy-type-checker", "mtxr.sqltools", "mtxr.sqltools-driver-pg", diff --git a/.vscode/settings.json b/.vscode/settings.json index 3d25dc7a..70e39139 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,4 +2,8 @@ "python.pylintPath": "/usr/local/py-utils/bin/pylint", "mypy-type-checker.cwd": "${workspaceFolder}/app", "pylint.cwd": "${workspaceFolder}/app", + "pylint.path": [ + "[\"../../usr/local/py-utils/bin/pylint\"]" + ], + "flake8.cwd": "${workspaceFolder}/app" } \ No newline at end of file From f2ec771a08d24729d306d2b7b04821d9ec7d7a93 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 17:43:46 +0000 Subject: [PATCH 08/34] feat: add a decorator to handle api calls errors --- .../google_workspace/google_service.py | 44 ++++++++++++++-- .../google_workspace/test_google_service.py | 50 ++++++++++++++++--- 2 files changed, 83 insertions(+), 11 deletions(-) diff --git a/app/integrations/google_workspace/google_service.py b/app/integrations/google_workspace/google_service.py index 587c7dad..b9eaf2c5 100644 --- a/app/integrations/google_workspace/google_service.py +++ b/app/integrations/google_workspace/google_service.py @@ -1,12 +1,26 @@ -"""Google Service Module.""" +""" +Google Service Module. + +This module provides a function to get an authenticated Google service and a decorator to handle Google API errors. + +Functions: + get_google_service(service: str, version: str) -> googleapiclient.discovery.Resource: + Returns an authenticated Google service resource for the specified service and version. + + handle_google_api_errors(func: Callable) -> Callable: + Decorator that catches and logs any HttpError or Error exceptions that occur when the decorated function is called. + +""" import os import logging import json from json import JSONDecodeError from dotenv import load_dotenv -from google.oauth2 import service_account -from googleapiclient.discovery import build +from functools import wraps +from google.oauth2 import service_account # type: ignore +from googleapiclient.discovery import build # type: ignore +from googleapiclient.errors import HttpError, Error # type: ignore load_dotenv() @@ -37,3 +51,27 @@ def get_google_service(service, version): msg="Invalid credentials JSON", doc="Credentials JSON", pos=0 ) from json_decode_exception return build(service, version, credentials=creds, cache_discovery=False) + + +def handle_google_api_errors(func): + """Decorator to handle Google API errors. + + Args: + func (function): The function to decorate. + + Returns: + The decorated function. + """ + + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except HttpError as e: + print(f"An HTTP error occurred: {e}") + return None + except Error as e: + print(f"An error occurred: {e}") + return None + + return wrapper diff --git a/app/tests/integrations/google_workspace/test_google_service.py b/app/tests/integrations/google_workspace/test_google_service.py index 4c24b91a..1053ea36 100644 --- a/app/tests/integrations/google_workspace/test_google_service.py +++ b/app/tests/integrations/google_workspace/test_google_service.py @@ -1,14 +1,18 @@ +"""Unit Tests for the google_service module.""" import json -import pytest from unittest.mock import patch, MagicMock -from integrations.google_workspace.google_service import get_google_service from json import JSONDecodeError +import pytest +from google.oauth2.service_account import Credentials +from integrations.google_workspace.google_service import ( + get_google_service, + handle_google_api_errors, +) +from googleapiclient.errors import HttpError, Error # type: ignore @patch("integrations.google_workspace.google_service.build") -@patch( - "integrations.google_workspace.google_service.service_account.Credentials.from_service_account_info" -) +@patch.object(Credentials, "from_service_account_info") def test_get_google_service_returns_build_object(credentials_mock, build_mock): """ Test case to verify that the function returns a build object. @@ -36,9 +40,7 @@ def test_get_google_service_raises_exception_if_credentials_json_not_set(): @patch("integrations.google_workspace.google_service.build") -@patch( - "integrations.google_workspace.google_service.service_account.Credentials.from_service_account_info" -) +@patch.object(Credentials, "from_service_account_info") def test_get_google_service_raises_exception_if_credentials_json_is_invalid( credentials_mock, build_mock ): @@ -50,3 +52,35 @@ def test_get_google_service_raises_exception_if_credentials_json_is_invalid( with pytest.raises(JSONDecodeError) as e: get_google_service("drive", "v3") assert "Invalid credentials JSON" in str(e.value) + + +def test_handle_google_api_errors_catches_http_error(): + mock_resp = MagicMock() + mock_resp.status = "400" + mock_func = MagicMock(side_effect=HttpError(resp=mock_resp, content=b"")) + decorated_func = handle_google_api_errors(mock_func) + + result = decorated_func() + + assert result is None + mock_func.assert_called_once() + + +def test_handle_google_api_errors_catches_error(): + mock_func = MagicMock(side_effect=Error()) + decorated_func = handle_google_api_errors(mock_func) + + result = decorated_func() + + assert result is None + mock_func.assert_called_once() + + +def test_handle_google_api_errors_passes_through_return_value(): + mock_func = MagicMock(return_value="test") + decorated_func = handle_google_api_errors(mock_func) + + result = decorated_func() + + assert result == "test" + mock_func.assert_called_once() From 2555afbd35cc87875a0a6665b3c1cf8333b2d036 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 18:52:09 +0000 Subject: [PATCH 09/34] feat: refactor basic drive functions --- .../google_workspace/google_drive.py | 220 ++++++++++++++++++ .../test_google_drive_module.py | 87 +++++++ 2 files changed, 307 insertions(+) create mode 100644 app/integrations/google_workspace/google_drive.py create mode 100644 app/tests/integrations/google_workspace/test_google_drive_module.py diff --git a/app/integrations/google_workspace/google_drive.py b/app/integrations/google_workspace/google_drive.py new file mode 100644 index 00000000..1097564e --- /dev/null +++ b/app/integrations/google_workspace/google_drive.py @@ -0,0 +1,220 @@ +""" +Google Drive Module. + +This module provides functionalities to interact with Google Drive. It includes functions to add metadata to a file, create a new folder, create a new document from a template, and copy a file to a new folder. + +Functions: + add_metadata(file_id: str, key: str, value: str) -> dict: + Adds metadata to a file in Google Drive and returns the updated file metadata. + + create_new_folder(name: str, parent_folder: str) -> str: + Creates a new folder in Google Drive and returns the id of the new folder. + + create_new_document_from_template(name: str, folder: str, template: str) -> str: + Creates a new document in Google Drive from a template (Docs, Sheets, Slides, Forms, or Sites) and returns the id of the new document. + + copy_file_to_folder(file_id: str, name: str, parent_folder_id: str, destination_folder_id: str) -> str: + Copies a file to a new folder in Google Drive and returns the id of the new file. +""" +from integrations.google_workspace.google_service import ( + get_google_service, + handle_google_api_errors, +) + + +@handle_google_api_errors +def add_metadata(file_id, key, value): + """Add metadata to a file in Google Drive. + + Args: + file_id (str): The file id of the file to add metadata to. + key (str): The key of the metadata to add. + value (str): The value of the metadata to add. + + Returns: + dict: The updated file metadata. + """ + # pylint: disable=no-member + service = get_google_service("drive", "v3") + result = ( + service.files() + .update( + fileId=file_id, + body={"appProperties": {key: value}}, + fields="name, appProperties", + supportsAllDrives=True, + ) + .execute() + ) + # pylint: enable=no-member + + return result + + +@handle_google_api_errors +def delete_metadata(file_id, key): + """Delete metadata from a file in Google Drive. + + Args: + file_id (str): The file id of the file to delete metadata from. + key (str): The key of the metadata to delete. + + Returns: + dict: The updated file metadata. + """ + service = get_google_service("drive", "v3") + result = ( + service.files() + .update( + fileId=file_id, + body={"appProperties": {key: None}}, + fields="name, appProperties", + supportsAllDrives=True, + ) + .execute() + ) + return result + + +@handle_google_api_errors +def create_folder(name, parent_folder): + """Create a new folder in Google Drive. + + Args: + name (str): The name of the new folder. + parent_folder (str): The id of the parent folder. + + Returns: + str: The id of the new folder. + """ + # pylint: disable=no-member + service = get_google_service("drive", "v3") + results = ( + service.files() + .create( + body={ + "name": name, + "mimeType": "application/vnd.google-apps.folder", + "parents": [parent_folder], + }, + supportsAllDrives=True, + fields="id", + ) + .execute() + ) + # pylint: enable=no-member + + return results["id"] + + +@handle_google_api_errors +def create_new_file_from_template(name, folder, template): + """Create a new file in Google Drive from a template + (Docs, Sheets, Slides, Forms, or Sites.) + + Args: + name (str): The name of the new file. + folder (str): The id of the folder to create the file in. + template (str): The id of the template to use. + + Returns: + str: The id of the new file. + """ + # pylint: disable=no-member + service = get_google_service("drive", "v3") + result = ( + service.files() + .copy( + fileId=template, + body={"name": name, "parents": [folder]}, + supportsAllDrives=True, + ) + .execute() + ) + # pylint: enable=no-member + return result["id"] + + +@handle_google_api_errors +def create_new_file(name, folder, file_type): + """Create a new file in Google Drive. + Options for 'file_type' are: + + - "document": Google Docs + - "spreadsheet": Google Sheets + - "presentation": Google Slides + - "form": Google Forms + - "site": Google Sites + + Args: + name (str): The name of the new file. + folder (str): The id of the folder to create the file in. + file_type (str): The type of the new file. + + Returns: + str: The id of the new file. + """ + + mime_type = { + "document": "application/vnd.google-apps.document", + "spreadsheet": "application/vnd.google-apps.spreadsheet", + "presentation": "application/vnd.google-apps.presentation", + "form": "application/vnd.google-apps.form", + "site": "application/vnd.google-apps.site", + } + + if file_type not in mime_type: + raise ValueError(f"Invalid file_type: {file_type}") + + mime_type_value = mime_type[file_type] + + service = get_google_service("drive", "v3") + result = ( + service.files() + .create( + body={"name": name, "parents": [folder], "mimeType": mime_type_value}, + supportsAllDrives=True, + fields="id", + ) + .execute() + ) + return result["id"] + + +@handle_google_api_errors +def copy_file_to_folder(file_id, name, parent_folder_id, destination_folder_id): + """Copy a file to a new folder in Google Drive. + + Args: + file_id (str): The id of the file to copy. + name (str): The name of the new file. + parent_folder_id (str): The id of the parent folder. + destination_folder_id (str): The id of the destination folder. + + Returns: + str: The id of the new file. + """ + service = get_google_service("drive", "v3") + copied_file = ( + service.files() + .copy( + fileId=file_id, + body={"name": name, "parents": [parent_folder_id]}, + supportsAllDrives=True, + fields="id", + ) + .execute() + ) + # move the copy to the new folder + updated_file = ( + service.files() + .update( + fileId=copied_file["id"], + addParents=destination_folder_id, + removeParents=parent_folder_id, + supportsAllDrives=True, + fields="id", + ) + .execute() + ) + return updated_file["id"] diff --git a/app/tests/integrations/google_workspace/test_google_drive_module.py b/app/tests/integrations/google_workspace/test_google_drive_module.py new file mode 100644 index 00000000..cd69a912 --- /dev/null +++ b/app/tests/integrations/google_workspace/test_google_drive_module.py @@ -0,0 +1,87 @@ +"""Unit tests for google_drive module.""" +import pytest +from unittest.mock import patch + +import integrations.google_workspace.google_drive as google_drive + +# Constants for the test +START_HEADING = "Detailed Timeline" +END_HEADING = "Trigger" + + +@patch("integrations.google_workspace.google_drive.get_google_service") +def test_add_metadata_returns_result(get_google_service_mock): + get_google_service_mock.return_value.files.return_value.update.return_value.execute.return_value = { + "name": "test_folder", + "appProperties": {"key": "value"}, + } + result = google_drive.add_metadata("file_id", "key", "value") + assert result == {"name": "test_folder", "appProperties": {"key": "value"}} + + +@patch("integrations.google_workspace.google_drive.get_google_service") +def test_delete_metadata_returns_result(get_google_service_mock): + get_google_service_mock.return_value.files.return_value.update.return_value.execute.return_value = { + "name": "test_folder", + "appProperties": {}, + } + result = google_drive.delete_metadata("file_id", "key") + get_google_service_mock.return_value.files.return_value.update.assert_called_once_with( + fileId="file_id", + body={"appProperties": {"key": None}}, + fields="name, appProperties", + supportsAllDrives=True, + ) + assert result == {"name": "test_folder", "appProperties": {}} + + +@patch("integrations.google_workspace.google_drive.get_google_service") +def test_create_folder_returns_folder_id(get_google_service_mock): + get_google_service_mock.return_value.files.return_value.create.return_value.execute.return_value = { + "id": "test_folder_id" + } + result = google_drive.create_folder("test_folder", "parent_folder") + assert result == "test_folder_id" + + +@patch("integrations.google_workspace.google_drive.get_google_service") +def test_create_new_file_with_valid_type_returns_file_id(get_google_service_mock): + get_google_service_mock.return_value.files.return_value.create.return_value.execute.return_value = { + "id": "test_document_id" + } + result = google_drive.create_new_file("test_document", "folder_id", "document") + assert result == "test_document_id" + + +@patch("integrations.google_workspace.google_drive.get_google_service") +def test_create_new_file_with_invalid_type_raises_value_error(get_google_service_mock): + with pytest.raises(ValueError) as e: + google_drive.create_new_file("test_document", "folder_id", "invalid_type") + assert "Invalid file_type: invalid_type" in str(e.value) + + +@patch("integrations.google_workspace.google_drive.get_google_service") +def test_create_new_file_from_template_returns_file_id(get_google_service_mock): + get_google_service_mock.return_value.files.return_value.copy.return_value.execute.return_value = { + "id": "test_document_id" + } + result = google_drive.create_new_file_from_template( + "test_document", "folder_id", "template_id" + ) + assert result == "test_document_id" + + +@patch("integrations.google_workspace.google_drive.get_google_service") +def test_copy_file_to_folder_returns_file_id(get_google_service_mock): + get_google_service_mock.return_value.files.return_value.copy.return_value.execute.return_value = { + "id": "file_id" + } + get_google_service_mock.return_value.files.return_value.update.return_value.execute.return_value = { + "id": "updated_file_id" + } + assert ( + google_drive.copy_file_to_folder( + "file_id", "name", "parent_folder", "destination_folder" + ) + == "updated_file_id" + ) From 6070afe6434a182474b356619ca6d333a3a5ee1e Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 19:02:36 +0000 Subject: [PATCH 10/34] feat: add get_file_by_name function --- .../google_workspace/google_drive.py | 28 +++++++++++++++++++ .../test_google_drive_module.py | 20 +++++++++++++ 2 files changed, 48 insertions(+) diff --git a/app/integrations/google_workspace/google_drive.py b/app/integrations/google_workspace/google_drive.py index 1097564e..048c4d36 100644 --- a/app/integrations/google_workspace/google_drive.py +++ b/app/integrations/google_workspace/google_drive.py @@ -181,6 +181,34 @@ def create_new_file(name, folder, file_type): return result["id"] +@handle_google_api_errors +def get_file_by_name(name, folder): + """Get a file by name in Google Drive. + + Args: + name (str): The name of the file to get. + folder (str): The id of the folder to search in. + + Returns: + list: A list of files that match the name. + """ + service = get_google_service("drive", "v3") + results = ( + service.files() + .list( + pageSize=1, + supportsAllDrives=True, + includeItemsFromAllDrives=True, + corpora="drive", + q="trashed=false and name='{}'".format(name), + driveId=folder, + fields="files(appProperties, id, name)", + ) + .execute() + ) + return results.get("files", []) + + @handle_google_api_errors def copy_file_to_folder(file_id, name, parent_folder_id, destination_folder_id): """Copy a file to a new folder in Google Drive. diff --git a/app/tests/integrations/google_workspace/test_google_drive_module.py b/app/tests/integrations/google_workspace/test_google_drive_module.py index cd69a912..be9fa543 100644 --- a/app/tests/integrations/google_workspace/test_google_drive_module.py +++ b/app/tests/integrations/google_workspace/test_google_drive_module.py @@ -71,6 +71,26 @@ def test_create_new_file_from_template_returns_file_id(get_google_service_mock): assert result == "test_document_id" +@patch("integrations.google_workspace.google_drive.get_google_service") +def test_get_file_by_name_returns_object(get_google_service_mock): + get_google_service_mock.return_value.files.return_value.list.return_value.execute.return_value = { + "files": [ + { + "name": "test_document", + "id": "test_document_id", + "appProperties": {}, + }, + ] + } + assert google_drive.get_file_by_name("test_file_name", "folder_id") == [ + { + "name": "test_document", + "id": "test_document_id", + "appProperties": {}, + } + ] + + @patch("integrations.google_workspace.google_drive.get_google_service") def test_copy_file_to_folder_returns_file_id(get_google_service_mock): get_google_service_mock.return_value.files.return_value.copy.return_value.execute.return_value = { From 97fa78ef5db902958c7161f1c9a0f6d411a653af Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 19:36:59 +0000 Subject: [PATCH 11/34] feat: add list metadata function --- .../google_workspace/google_drive.py | 47 +++++++++++++++++++ .../test_google_drive_module.py | 20 ++++++++ 2 files changed, 67 insertions(+) diff --git a/app/integrations/google_workspace/google_drive.py b/app/integrations/google_workspace/google_drive.py index 048c4d36..c2a8f4b3 100644 --- a/app/integrations/google_workspace/google_drive.py +++ b/app/integrations/google_workspace/google_drive.py @@ -76,6 +76,25 @@ def delete_metadata(file_id, key): return result +@handle_google_api_errors +def list_metadata(file_id): + """List metadata of a file in Google Drive. + + Args: + file_id (str): The file id of the file to list metadata from. + + Returns: + dict: The file metadata. + """ + service = get_google_service("drive", "v3") + result = ( + service.files() + .get(fileId=file_id, fields="name, appProperties", supportsAllDrives=True) + .execute() + ) + return result + + @handle_google_api_errors def create_folder(name, parent_folder): """Create a new folder in Google Drive. @@ -209,6 +228,34 @@ def get_file_by_name(name, folder): return results.get("files", []) +@handle_google_api_errors +def list_folders_in_folder(folder): + """List all folders in a folder in Google Drive. + + Args: + folder (str): The id of the folder to list. + + Returns: + list: A list of folders in the folder. + """ + service = get_google_service("drive", "v3") + results = ( + service.files() + .list( + pageSize=25, + supportsAllDrives=True, + includeItemsFromAllDrives=True, + corpora="user", + q="parents in '{}' and mimeType = 'application/vnd.google-apps.folder' and trashed=false".format( + folder + ), + fields="nextPageToken, files(id, name)", + ) + .execute() + ) + return results.get("files", []) + + @handle_google_api_errors def copy_file_to_folder(file_id, name, parent_folder_id, destination_folder_id): """Copy a file to a new folder in Google Drive. diff --git a/app/tests/integrations/google_workspace/test_google_drive_module.py b/app/tests/integrations/google_workspace/test_google_drive_module.py index be9fa543..8457f6e4 100644 --- a/app/tests/integrations/google_workspace/test_google_drive_module.py +++ b/app/tests/integrations/google_workspace/test_google_drive_module.py @@ -35,6 +35,16 @@ def test_delete_metadata_returns_result(get_google_service_mock): assert result == {"name": "test_folder", "appProperties": {}} +@patch("integrations.google_workspace.google_drive.get_google_service") +def test_list_metadata_returns_result(get_google_service_mock): + get_google_service_mock.return_value.files.return_value.get.return_value.execute.return_value = { + "name": "test_folder", + "appProperties": {"key": "value"}, + } + result = google_drive.list_metadata("file_id") + assert result == {"name": "test_folder", "appProperties": {"key": "value"}} + + @patch("integrations.google_workspace.google_drive.get_google_service") def test_create_folder_returns_folder_id(get_google_service_mock): get_google_service_mock.return_value.files.return_value.create.return_value.execute.return_value = { @@ -105,3 +115,13 @@ def test_copy_file_to_folder_returns_file_id(get_google_service_mock): ) == "updated_file_id" ) + + +@patch("integrations.google_workspace.google_drive.get_google_service") +def test_list_folders_returns_folder_names(get_google_service_mock): + get_google_service_mock.return_value.files.return_value.list.return_value.execute.return_value = { + "files": [{"name": "test_folder"}] + } + assert google_drive.list_folders_in_folder("parent_folder") == [{"name": "test_folder"}] + + From f875a9bcee46256c3ecc2f211b3ae27e964205f1 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 19:45:00 +0000 Subject: [PATCH 12/34] feat: use new drive module functions --- app/commands/google_service.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/app/commands/google_service.py b/app/commands/google_service.py index a0609758..a2fb7ed6 100644 --- a/app/commands/google_service.py +++ b/app/commands/google_service.py @@ -1,7 +1,8 @@ """Testing new google service (will be removed)""" import os -from integrations.google_workspace.google_service import get_google_service +# from integrations.google_workspace.google_service import get_google_service +from integrations.google_workspace import google_drive from dotenv import load_dotenv load_dotenv() @@ -30,20 +31,4 @@ def google_service_command(client, body): def list_folders(): - service = get_google_service("drive", "v3") - results = ( - service.files() - .list( - pageSize=25, - supportsAllDrives=True, - includeItemsFromAllDrives=True, - corpora="drive", - q="parents in '{}' and mimeType = 'application/vnd.google-apps.folder' and trashed=false and not name contains '{}'".format( - SRE_INCIDENT_FOLDER, "Templates" - ), - driveId=SRE_DRIVE_ID, - fields="nextPageToken, files(id, name)", - ) - .execute() - ) - return results.get("files", []) + return google_drive.list_folders_in_folder(SRE_INCIDENT_FOLDER) From c087c4180c8e4ddea4aa342dd959294868416cb4 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 20:07:22 +0000 Subject: [PATCH 13/34] fix: shorten test command --- app/commands/sre.py | 4 ++-- app/main.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/commands/sre.py b/app/commands/sre.py index 7749ae19..da3e43d5 100644 --- a/app/commands/sre.py +++ b/app/commands/sre.py @@ -49,9 +49,9 @@ def sre_command(ack, command, logger, respond, client, body): webhook_helper.handle_webhook_command(args, client, body, respond) case "version": respond(f"SRE Bot version: {os.environ.get('GIT_SHA', 'unknown')}") - case "google-service": + case "google": if PREFIX == "dev-": - google_service.google_service_command(client, body) + google_service.google_service_command(client, body, respond) else: respond("This command is only available in the dev environment.") return diff --git a/app/main.py b/app/main.py index 482c900f..6cbf48b6 100644 --- a/app/main.py +++ b/app/main.py @@ -96,7 +96,7 @@ def main(bot): # Register Google Service command for dev purposes only if PREFIX == "dev-": - bot.command(f"/{PREFIX}google-service")(google_service.google_service_command) + bot.command(f"/{PREFIX}google")(google_service.google_service_command) bot.view("google_service_view")(google_service.open_modal) From cf653333213bb5e003c1940be63e479e0b053dc7 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 20:07:45 +0000 Subject: [PATCH 14/34] fix: handle invalid folder --- app/commands/google_service.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/app/commands/google_service.py b/app/commands/google_service.py index a2fb7ed6..016f33fc 100644 --- a/app/commands/google_service.py +++ b/app/commands/google_service.py @@ -11,8 +11,9 @@ SRE_INCIDENT_FOLDER = os.environ.get("SRE_INCIDENT_FOLDER") -def open_modal(client, body): - folders = list_folders() +def open_modal(client, body, folders): + if not folders: + return folder_names = [i["name"] for i in folders] blocks = [ {"type": "section", "text": {"type": "mrkdwn", "text": f"*{name}*"}} @@ -26,9 +27,9 @@ def open_modal(client, body): client.views_open(trigger_id=body["trigger_id"], view=view) -def google_service_command(client, body): - open_modal(client, body) - - -def list_folders(): - return google_drive.list_folders_in_folder(SRE_INCIDENT_FOLDER) +def google_service_command(client, body, respond): + folders = google_drive.list_folders_in_folder(SRE_INCIDENT_FOLDER) + if not folders: + respond("The folder ID is invalid. Please check the environment variables.") + return + open_modal(client, body, folders) From b1b5de11103ba62184be1439534104699c36837c Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 20:37:01 +0000 Subject: [PATCH 15/34] feat: modify wrapper to provide context --- app/integrations/google_workspace/google_service.py | 4 ++-- .../google_workspace/test_google_service.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/app/integrations/google_workspace/google_service.py b/app/integrations/google_workspace/google_service.py index b9eaf2c5..3f7a15d6 100644 --- a/app/integrations/google_workspace/google_service.py +++ b/app/integrations/google_workspace/google_service.py @@ -68,10 +68,10 @@ def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except HttpError as e: - print(f"An HTTP error occurred: {e}") + print(f"An HTTP error occurred in function {func.__name__}: {e}") return None except Error as e: - print(f"An error occurred: {e}") + print(f"An error occurred in function {func.__name__}: {e}") return None return wrapper diff --git a/app/tests/integrations/google_workspace/test_google_service.py b/app/tests/integrations/google_workspace/test_google_service.py index 1053ea36..833cee2a 100644 --- a/app/tests/integrations/google_workspace/test_google_service.py +++ b/app/tests/integrations/google_workspace/test_google_service.py @@ -54,26 +54,32 @@ def test_get_google_service_raises_exception_if_credentials_json_is_invalid( assert "Invalid credentials JSON" in str(e.value) -def test_handle_google_api_errors_catches_http_error(): +def test_handle_google_api_errors_catches_http_error(capfd): mock_resp = MagicMock() mock_resp.status = "400" mock_func = MagicMock(side_effect=HttpError(resp=mock_resp, content=b"")) + mock_func.__name__ = "mock_func" decorated_func = handle_google_api_errors(mock_func) result = decorated_func() assert result is None mock_func.assert_called_once() + out, err = capfd.readouterr() + assert "An HTTP error occurred in function mock_func:" in out -def test_handle_google_api_errors_catches_error(): +def test_handle_google_api_errors_catches_error(capfd): mock_func = MagicMock(side_effect=Error()) + mock_func.__name__ = "mock_func" decorated_func = handle_google_api_errors(mock_func) result = decorated_func() assert result is None mock_func.assert_called_once() + out, err = capfd.readouterr() + assert "An error occurred in function mock_func:" in out def test_handle_google_api_errors_passes_through_return_value(): From 463464dbcd643ee32c996930223c70a42f4a66db Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 20:54:48 +0000 Subject: [PATCH 16/34] fix: add missing parameter to list_metadata --- .../google_workspace/google_drive.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/app/integrations/google_workspace/google_drive.py b/app/integrations/google_workspace/google_drive.py index c2a8f4b3..dad38f62 100644 --- a/app/integrations/google_workspace/google_drive.py +++ b/app/integrations/google_workspace/google_drive.py @@ -16,11 +16,15 @@ copy_file_to_folder(file_id: str, name: str, parent_folder_id: str, destination_folder_id: str) -> str: Copies a file to a new folder in Google Drive and returns the id of the new file. """ +import os from integrations.google_workspace.google_service import ( get_google_service, handle_google_api_errors, ) +SRE_INCIDENT_FOLDER = os.environ.get("SRE_INCIDENT_FOLDER") +INCIDENT_TEMPLATE = os.environ.get("INCIDENT_TEMPLATE") + @handle_google_api_errors def add_metadata(file_id, key, value): @@ -89,7 +93,7 @@ def list_metadata(file_id): service = get_google_service("drive", "v3") result = ( service.files() - .get(fileId=file_id, fields="name, appProperties", supportsAllDrives=True) + .get(fileId=file_id, fields="id, name, appProperties", supportsAllDrives=True) .execute() ) return result @@ -293,3 +297,17 @@ def copy_file_to_folder(file_id, name, parent_folder_id, destination_folder_id): .execute() ) return updated_file["id"] + + +@handle_google_api_errors +def healthcheck(): + """Check the health of the Google Drive API. + + Returns: + bool: True if the API is healthy, False otherwise. + """ + healthy = False + metadata = list_metadata(INCIDENT_TEMPLATE) + healthy = "id" in metadata + + return healthy From 83f2fa61ebb99b1c8c59fd92edb1547077d85d56 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 20:55:10 +0000 Subject: [PATCH 17/34] fix: surround function name w/ single quotes --- app/integrations/google_workspace/google_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/integrations/google_workspace/google_service.py b/app/integrations/google_workspace/google_service.py index 3f7a15d6..29d7eb51 100644 --- a/app/integrations/google_workspace/google_service.py +++ b/app/integrations/google_workspace/google_service.py @@ -68,10 +68,10 @@ def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except HttpError as e: - print(f"An HTTP error occurred in function {func.__name__}: {e}") + print(f"An HTTP error occurred in function '{func.__name__}': {e}") return None except Error as e: - print(f"An error occurred in function {func.__name__}: {e}") + print(f"An error occurred in function '{func.__name__}': {e}") return None return wrapper From fead205c0e3bbadf72cdb10972d2532f83d07ed0 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 21:19:56 +0000 Subject: [PATCH 18/34] fix: google drive healthcheck --- app/integrations/google_workspace/google_drive.py | 3 ++- .../google_workspace/test_google_drive_module.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/integrations/google_workspace/google_drive.py b/app/integrations/google_workspace/google_drive.py index dad38f62..63a87354 100644 --- a/app/integrations/google_workspace/google_drive.py +++ b/app/integrations/google_workspace/google_drive.py @@ -308,6 +308,7 @@ def healthcheck(): """ healthy = False metadata = list_metadata(INCIDENT_TEMPLATE) - healthy = "id" in metadata + if metadata is not None: + healthy = "id" in metadata return healthy diff --git a/app/tests/integrations/google_workspace/test_google_drive_module.py b/app/tests/integrations/google_workspace/test_google_drive_module.py index 8457f6e4..1bc38b1e 100644 --- a/app/tests/integrations/google_workspace/test_google_drive_module.py +++ b/app/tests/integrations/google_workspace/test_google_drive_module.py @@ -122,6 +122,18 @@ def test_list_folders_returns_folder_names(get_google_service_mock): get_google_service_mock.return_value.files.return_value.list.return_value.execute.return_value = { "files": [{"name": "test_folder"}] } - assert google_drive.list_folders_in_folder("parent_folder") == [{"name": "test_folder"}] + assert google_drive.list_folders_in_folder("parent_folder") == [ + {"name": "test_folder"} + ] + + +@patch("integrations.google_workspace.google_drive.list_metadata") +def test_healthcheck_healthy(list_metadata_mock): + list_metadata_mock.return_value = {"id": "test_doc"} + assert google_drive.healthcheck() is True +@patch("integrations.google_workspace.google_drive.list_metadata") +def test_healthcheck_unhealthy(list_metadata_mock): + list_metadata_mock.return_value = None + assert google_drive.healthcheck() is False From 61d8c68b324a4ea49f3dd2994fafc5c64e4371c1 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 21:53:20 +0000 Subject: [PATCH 19/34] fix: matching error message --- .../integrations/google_workspace/test_google_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tests/integrations/google_workspace/test_google_service.py b/app/tests/integrations/google_workspace/test_google_service.py index 833cee2a..483b2cc5 100644 --- a/app/tests/integrations/google_workspace/test_google_service.py +++ b/app/tests/integrations/google_workspace/test_google_service.py @@ -66,7 +66,7 @@ def test_handle_google_api_errors_catches_http_error(capfd): assert result is None mock_func.assert_called_once() out, err = capfd.readouterr() - assert "An HTTP error occurred in function mock_func:" in out + assert "An HTTP error occurred in function 'mock_func':" in out def test_handle_google_api_errors_catches_error(capfd): @@ -79,7 +79,7 @@ def test_handle_google_api_errors_catches_error(capfd): assert result is None mock_func.assert_called_once() out, err = capfd.readouterr() - assert "An error occurred in function mock_func:" in out + assert "An error occurred in function 'mock_func':" in out def test_handle_google_api_errors_passes_through_return_value(): From 001172216af79c60259035dda146ae6814af529b Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Thu, 1 Feb 2024 21:53:51 +0000 Subject: [PATCH 20/34] feat: add healtcheck message --- app/commands/google_service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/commands/google_service.py b/app/commands/google_service.py index 016f33fc..bcb03d40 100644 --- a/app/commands/google_service.py +++ b/app/commands/google_service.py @@ -9,6 +9,7 @@ SRE_DRIVE_ID = os.environ.get("SRE_DRIVE_ID") SRE_INCIDENT_FOLDER = os.environ.get("SRE_INCIDENT_FOLDER") +INCIDENT_TEMPLATE = os.environ.get("INCIDENT_TEMPLATE") def open_modal(client, body, folders): @@ -28,6 +29,7 @@ def open_modal(client, body, folders): def google_service_command(client, body, respond): + respond(f"Healthcheck status: {google_drive.healthcheck()}") folders = google_drive.list_folders_in_folder(SRE_INCIDENT_FOLDER) if not folders: respond("The folder ID is invalid. Please check the environment variables.") From 890ff081ba5706dd7f600280881e61c400e4127f Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 01:03:46 +0000 Subject: [PATCH 21/34] fix: update module docstrings w/ more details --- app/integrations/google_workspace/google_meet.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/integrations/google_workspace/google_meet.py b/app/integrations/google_workspace/google_meet.py index a481f76f..b8bf83db 100644 --- a/app/integrations/google_workspace/google_meet.py +++ b/app/integrations/google_workspace/google_meet.py @@ -1,4 +1,11 @@ -"""Google Meet integration.""" +"""Google Meet Module. + +This module provides a function to start a Google Meet session. + +Functions: + create_google_meet(title: str) -> str: + Starts a Google Meet session and returns the URL of the session. +""" import re from datetime import datetime From 6c344f66b3d5b3c283831a48366150850270552a Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 01:05:43 +0000 Subject: [PATCH 22/34] fix: remove import from google_service --- app/commands/google_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/commands/google_service.py b/app/commands/google_service.py index bcb03d40..21e0ca53 100644 --- a/app/commands/google_service.py +++ b/app/commands/google_service.py @@ -1,7 +1,6 @@ """Testing new google service (will be removed)""" import os -# from integrations.google_workspace.google_service import get_google_service from integrations.google_workspace import google_drive from dotenv import load_dotenv From 02788a73a19b1c8dc05f69dc227d8bbac7da2a69 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 01:26:24 +0000 Subject: [PATCH 23/34] fix: update module docstring --- .../google_workspace/google_drive.py | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/app/integrations/google_workspace/google_drive.py b/app/integrations/google_workspace/google_drive.py index 63a87354..c1786c37 100644 --- a/app/integrations/google_workspace/google_drive.py +++ b/app/integrations/google_workspace/google_drive.py @@ -7,14 +7,32 @@ add_metadata(file_id: str, key: str, value: str) -> dict: Adds metadata to a file in Google Drive and returns the updated file metadata. - create_new_folder(name: str, parent_folder: str) -> str: + delete_metadata(file_id: str, key: str) -> dict: + Deletes metadata from a file in Google Drive and returns the updated file metadata. + + list_metadata(file_id: str) -> dict: + Lists metadata of a file in Google Drive and returns the file metadata. + + create_folder(name: str, parent_folder: str) -> str: Creates a new folder in Google Drive and returns the id of the new folder. - create_new_document_from_template(name: str, folder: str, template: str) -> str: + create_file_from_template(name: str, folder: str, template: str) -> str: Creates a new document in Google Drive from a template (Docs, Sheets, Slides, Forms, or Sites) and returns the id of the new document. + create_file(name: str, folder: str, file_type: str) -> str: + Creates a new file in Google Drive and returns the id of the new file. + + get_file_by_name(name: str, folder: str) -> list: + Gets a file by name in Google Drive and returns a list of files that match the name. + + list_folders_in_folder(folder: str) -> list: + Lists all folders in a folder in Google Drive and returns a list of folders in the folder. + copy_file_to_folder(file_id: str, name: str, parent_folder_id: str, destination_folder_id: str) -> str: Copies a file to a new folder in Google Drive and returns the id of the new file. + + healthcheck() -> bool: + Checks the health of the Google Drive API and returns True if the API is healthy, False otherwise. """ import os from integrations.google_workspace.google_service import ( @@ -22,7 +40,6 @@ handle_google_api_errors, ) -SRE_INCIDENT_FOLDER = os.environ.get("SRE_INCIDENT_FOLDER") INCIDENT_TEMPLATE = os.environ.get("INCIDENT_TEMPLATE") @@ -131,7 +148,7 @@ def create_folder(name, parent_folder): @handle_google_api_errors -def create_new_file_from_template(name, folder, template): +def create_file_from_template(name, folder, template): """Create a new file in Google Drive from a template (Docs, Sheets, Slides, Forms, or Sites.) @@ -159,7 +176,7 @@ def create_new_file_from_template(name, folder, template): @handle_google_api_errors -def create_new_file(name, folder, file_type): +def create_file(name, folder, file_type): """Create a new file in Google Drive. Options for 'file_type' are: From 332a5feef568878f17da0cb8f06d5049988087f3 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 01:26:45 +0000 Subject: [PATCH 24/34] feat: add basic docstring for Docs module --- app/integrations/google_workspace/google_docs.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 app/integrations/google_workspace/google_docs.py diff --git a/app/integrations/google_workspace/google_docs.py b/app/integrations/google_workspace/google_docs.py new file mode 100644 index 00000000..275ae601 --- /dev/null +++ b/app/integrations/google_workspace/google_docs.py @@ -0,0 +1,4 @@ +"""Google Docs module. + +This module provides functions to create and manipulate Google Docs. +""" From aa88af74be723dbd3f2cfa0f13545a8d2e832425 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 01:27:03 +0000 Subject: [PATCH 25/34] fix: replicate current tests for conformity --- .../test_google_drive_module.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/app/tests/integrations/google_workspace/test_google_drive_module.py b/app/tests/integrations/google_workspace/test_google_drive_module.py index 1bc38b1e..c2d1a923 100644 --- a/app/tests/integrations/google_workspace/test_google_drive_module.py +++ b/app/tests/integrations/google_workspace/test_google_drive_module.py @@ -41,8 +41,10 @@ def test_list_metadata_returns_result(get_google_service_mock): "name": "test_folder", "appProperties": {"key": "value"}, } - result = google_drive.list_metadata("file_id") - assert result == {"name": "test_folder", "appProperties": {"key": "value"}} + assert google_drive.list_metadata("file_id") == { + "name": "test_folder", + "appProperties": {"key": "value"}, + } @patch("integrations.google_workspace.google_drive.get_google_service") @@ -50,8 +52,9 @@ def test_create_folder_returns_folder_id(get_google_service_mock): get_google_service_mock.return_value.files.return_value.create.return_value.execute.return_value = { "id": "test_folder_id" } - result = google_drive.create_folder("test_folder", "parent_folder") - assert result == "test_folder_id" + assert ( + google_drive.create_folder("test_folder", "parent_folder") == "test_folder_id" + ) @patch("integrations.google_workspace.google_drive.get_google_service") @@ -59,14 +62,14 @@ def test_create_new_file_with_valid_type_returns_file_id(get_google_service_mock get_google_service_mock.return_value.files.return_value.create.return_value.execute.return_value = { "id": "test_document_id" } - result = google_drive.create_new_file("test_document", "folder_id", "document") + result = google_drive.create_file("test_document", "folder_id", "document") assert result == "test_document_id" @patch("integrations.google_workspace.google_drive.get_google_service") def test_create_new_file_with_invalid_type_raises_value_error(get_google_service_mock): with pytest.raises(ValueError) as e: - google_drive.create_new_file("test_document", "folder_id", "invalid_type") + google_drive.create_file("test_document", "folder_id", "invalid_type") assert "Invalid file_type: invalid_type" in str(e.value) @@ -75,7 +78,7 @@ def test_create_new_file_from_template_returns_file_id(get_google_service_mock): get_google_service_mock.return_value.files.return_value.copy.return_value.execute.return_value = { "id": "test_document_id" } - result = google_drive.create_new_file_from_template( + result = google_drive.create_file_from_template( "test_document", "folder_id", "template_id" ) assert result == "test_document_id" From 24c101e0ff08ec1fa2f6a1a97726c34c4e8fa01d Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 09:43:40 -0500 Subject: [PATCH 26/34] fix: add loop to handle pagination Co-authored-by: Pat Heard --- .../google_workspace/google_drive.py | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/app/integrations/google_workspace/google_drive.py b/app/integrations/google_workspace/google_drive.py index c1786c37..dfc4da78 100644 --- a/app/integrations/google_workspace/google_drive.py +++ b/app/integrations/google_workspace/google_drive.py @@ -260,20 +260,28 @@ def list_folders_in_folder(folder): list: A list of folders in the folder. """ service = get_google_service("drive", "v3") - results = ( - service.files() - .list( - pageSize=25, - supportsAllDrives=True, - includeItemsFromAllDrives=True, - corpora="user", - q="parents in '{}' and mimeType = 'application/vnd.google-apps.folder' and trashed=false".format( - folder - ), - fields="nextPageToken, files(id, name)", + page_token = None + all_files = [] + while True: + results = ( + service.files() + .list( + pageSize=25, + supportsAllDrives=True, + includeItemsFromAllDrives=True, + corpora="user", + q="parents in '{}' and mimeType = 'application/vnd.google-apps.folder' and trashed=false".format( + folder + ), + fields="nextPageToken, files(id, name)", + pageToken=page_token, + ) + .execute() ) - .execute() - ) + all_files.extend(results.get('files', [])) + page_token = results.get('nextPageToken') + if not page_token: + break return results.get("files", []) From 2a0a9933266159de077c4348cd509df0c2b43cda Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:06:11 +0000 Subject: [PATCH 27/34] fix: ensure all files are returned --- app/integrations/google_workspace/google_drive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/integrations/google_workspace/google_drive.py b/app/integrations/google_workspace/google_drive.py index dfc4da78..ab4d0c05 100644 --- a/app/integrations/google_workspace/google_drive.py +++ b/app/integrations/google_workspace/google_drive.py @@ -278,11 +278,11 @@ def list_folders_in_folder(folder): ) .execute() ) - all_files.extend(results.get('files', [])) - page_token = results.get('nextPageToken') + all_files.extend(results.get("files", [])) + page_token = results.get("nextPageToken") if not page_token: break - return results.get("files", []) + return all_files @handle_google_api_errors From 68a262ca84bbf0d4ccf38a6e1e63d7d8dd8a8b4a Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:06:26 +0000 Subject: [PATCH 28/34] fix: add test to handle pagination --- .../google_workspace/test_google_drive_module.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/tests/integrations/google_workspace/test_google_drive_module.py b/app/tests/integrations/google_workspace/test_google_drive_module.py index c2d1a923..0dffbae0 100644 --- a/app/tests/integrations/google_workspace/test_google_drive_module.py +++ b/app/tests/integrations/google_workspace/test_google_drive_module.py @@ -130,6 +130,18 @@ def test_list_folders_returns_folder_names(get_google_service_mock): ] +@patch("integrations.google_workspace.google_drive.get_google_service") +def test_list_folders_iterates_over_pages(get_google_service_mock): + get_google_service_mock.return_value.files.return_value.list.return_value.execute.side_effect = [ + {"files": [{"name": "test_folder"}], "nextPageToken": "token"}, + {"files": [{"name": "test_folder2"}]}, + ] + assert google_drive.list_folders_in_folder("parent_folder") == [ + {"name": "test_folder"}, + {"name": "test_folder2"}, + ] + + @patch("integrations.google_workspace.google_drive.list_metadata") def test_healthcheck_healthy(list_metadata_mock): list_metadata_mock.return_value = {"id": "test_doc"} From 129fcdd12feb95b062030aaf61935bd9713a53cb Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:37:00 +0000 Subject: [PATCH 29/34] fix: log errors --- app/integrations/google_workspace/google_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/integrations/google_workspace/google_service.py b/app/integrations/google_workspace/google_service.py index 29d7eb51..321c13fb 100644 --- a/app/integrations/google_workspace/google_service.py +++ b/app/integrations/google_workspace/google_service.py @@ -68,10 +68,10 @@ def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except HttpError as e: - print(f"An HTTP error occurred in function '{func.__name__}': {e}") + logging.error(f"An HTTP error occurred in function '{func.__name__}': {e}") return None except Error as e: - print(f"An error occurred in function '{func.__name__}': {e}") + logging.error(f"An error occurred in function '{func.__name__}': {e}") return None return wrapper From dab733e5ad29a2acb3faef37d6795edb8eb082d0 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:37:16 +0000 Subject: [PATCH 30/34] fix: remove unused variables --- .../integrations/google_workspace/test_google_drive_module.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/tests/integrations/google_workspace/test_google_drive_module.py b/app/tests/integrations/google_workspace/test_google_drive_module.py index 0dffbae0..f75a29bf 100644 --- a/app/tests/integrations/google_workspace/test_google_drive_module.py +++ b/app/tests/integrations/google_workspace/test_google_drive_module.py @@ -4,10 +4,6 @@ import integrations.google_workspace.google_drive as google_drive -# Constants for the test -START_HEADING = "Detailed Timeline" -END_HEADING = "Trigger" - @patch("integrations.google_workspace.google_drive.get_google_service") def test_add_metadata_returns_result(get_google_service_mock): From 40d441356104b67598fc122c401da9f9af2bf796 Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 16:38:51 +0000 Subject: [PATCH 31/34] feat: include ValueError logging in the wrapper function --- app/integrations/google_workspace/google_service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/integrations/google_workspace/google_service.py b/app/integrations/google_workspace/google_service.py index 321c13fb..6f375371 100644 --- a/app/integrations/google_workspace/google_service.py +++ b/app/integrations/google_workspace/google_service.py @@ -70,6 +70,9 @@ def wrapper(*args, **kwargs): except HttpError as e: logging.error(f"An HTTP error occurred in function '{func.__name__}': {e}") return None + except ValueError as e: + logging.error(f"A ValueError occurred in function '{func.__name__}': {e}") + return None except Error as e: logging.error(f"An error occurred in function '{func.__name__}': {e}") return None From 3b7396812c3017be11a71c1e498e1b842d1400de Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 16:39:52 +0000 Subject: [PATCH 32/34] fix: adjust tests to handle logging errors --- .../google_workspace/test_google_service.py | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/app/tests/integrations/google_workspace/test_google_service.py b/app/tests/integrations/google_workspace/test_google_service.py index 483b2cc5..efee4cc3 100644 --- a/app/tests/integrations/google_workspace/test_google_service.py +++ b/app/tests/integrations/google_workspace/test_google_service.py @@ -54,23 +54,41 @@ def test_get_google_service_raises_exception_if_credentials_json_is_invalid( assert "Invalid credentials JSON" in str(e.value) -def test_handle_google_api_errors_catches_http_error(capfd): +@patch("logging.error") +def test_handle_google_api_errors_catches_http_error(mocked_logging_error): mock_resp = MagicMock() mock_resp.status = "400" + mock_resp.reason = "Bad Request" mock_func = MagicMock(side_effect=HttpError(resp=mock_resp, content=b"")) mock_func.__name__ = "mock_func" decorated_func = handle_google_api_errors(mock_func) result = decorated_func() + assert result is None + mocked_logging_error.assert_called_once_with( + "An HTTP error occurred in function 'mock_func': " + ) + + +@patch("logging.error") +def test_handle_google_api_errors_catches_value_error(mocked_logging_error): + mock_func = MagicMock(side_effect=ValueError("ValueError message")) + mock_func.__name__ = "mock_func" + decorated_func = handle_google_api_errors(mock_func) + + result = decorated_func() + assert result is None mock_func.assert_called_once() - out, err = capfd.readouterr() - assert "An HTTP error occurred in function 'mock_func':" in out + mocked_logging_error.assert_called_once_with( + "A ValueError occurred in function 'mock_func': ValueError message" + ) -def test_handle_google_api_errors_catches_error(capfd): - mock_func = MagicMock(side_effect=Error()) +@patch("logging.error") +def test_handle_google_api_errors_catches_error(mocked_logging_error): + mock_func = MagicMock(side_effect=Error("Error message")) mock_func.__name__ = "mock_func" decorated_func = handle_google_api_errors(mock_func) @@ -78,8 +96,9 @@ def test_handle_google_api_errors_catches_error(capfd): assert result is None mock_func.assert_called_once() - out, err = capfd.readouterr() - assert "An error occurred in function 'mock_func':" in out + mocked_logging_error.assert_called_once_with( + "An error occurred in function 'mock_func': Error message" + ) def test_handle_google_api_errors_passes_through_return_value(): From 4ad41da5558043ef5d825938286bf14cdd39bcba Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 16:47:32 +0000 Subject: [PATCH 33/34] fix: test names --- .../test_google_drive_module.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/app/tests/integrations/google_workspace/test_google_drive_module.py b/app/tests/integrations/google_workspace/test_google_drive_module.py index f75a29bf..a44acf55 100644 --- a/app/tests/integrations/google_workspace/test_google_drive_module.py +++ b/app/tests/integrations/google_workspace/test_google_drive_module.py @@ -54,7 +54,7 @@ def test_create_folder_returns_folder_id(get_google_service_mock): @patch("integrations.google_workspace.google_drive.get_google_service") -def test_create_new_file_with_valid_type_returns_file_id(get_google_service_mock): +def test_create_file_with_valid_type_returns_file_id(get_google_service_mock): get_google_service_mock.return_value.files.return_value.create.return_value.execute.return_value = { "id": "test_document_id" } @@ -62,15 +62,21 @@ def test_create_new_file_with_valid_type_returns_file_id(get_google_service_mock assert result == "test_document_id" +@patch("logging.error") @patch("integrations.google_workspace.google_drive.get_google_service") -def test_create_new_file_with_invalid_type_raises_value_error(get_google_service_mock): - with pytest.raises(ValueError) as e: - google_drive.create_file("test_document", "folder_id", "invalid_type") - assert "Invalid file_type: invalid_type" in str(e.value) +def test_create_file_with_invalid_type_raises_value_error( + get_google_service_mock, mocked_logging_error +): + result = google_drive.create_file("name", "folder", "invalid_file_type") + + assert result is None + mocked_logging_error.assert_called_once_with( + "A ValueError occurred in function 'create_file': Invalid file_type: invalid_file_type" + ) @patch("integrations.google_workspace.google_drive.get_google_service") -def test_create_new_file_from_template_returns_file_id(get_google_service_mock): +def test_create_file_from_template_returns_file_id(get_google_service_mock): get_google_service_mock.return_value.files.return_value.copy.return_value.execute.return_value = { "id": "test_document_id" } From dda6090e8f1b996deaa242d31220fff400c9eddd Mon Sep 17 00:00:00 2001 From: Guillaume Charest <1690085+gcharest@users.noreply.github.com> Date: Fri, 2 Feb 2024 16:48:30 +0000 Subject: [PATCH 34/34] fix: lint --- .../integrations/google_workspace/test_google_drive_module.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/tests/integrations/google_workspace/test_google_drive_module.py b/app/tests/integrations/google_workspace/test_google_drive_module.py index a44acf55..5d689601 100644 --- a/app/tests/integrations/google_workspace/test_google_drive_module.py +++ b/app/tests/integrations/google_workspace/test_google_drive_module.py @@ -1,5 +1,4 @@ """Unit tests for google_drive module.""" -import pytest from unittest.mock import patch import integrations.google_workspace.google_drive as google_drive