From 6098b5a6905a95cd3ec3a5ad0c6773a9be0820d6 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 27 Jun 2024 10:17:12 +0200 Subject: [PATCH 01/17] wip --- google_sheets/app.py | 92 +++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index 2088496..8cec05f 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -1,9 +1,10 @@ import json import logging from os import environ -from typing import Annotated, Dict, List, Union +from typing import Annotated, Dict, List, Literal, Union import httpx +import pandas as pd from fastapi import FastAPI, HTTPException, Query, Request, Response, status from fastapi.responses import RedirectResponse from googleapiclient.errors import HttpError @@ -265,3 +266,92 @@ async def get_all_sheet_titles( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) ) from e return sheets + + +NEW_CAMPAIGN_MANDATORY_COLUMNS = ["Country", "Station From", "Station To"] +MANDATORY_TEMPLATE_COLUMNS = [ + "Campaign", + "Ad Group", + "Headline 1", + "Headline 2", + "Headline 3", + "Description Line 1", + "Description Line 2", + "Final Url", +] + + +def process_ad_data( + template_df: pd.DataFrame, new_campaign_df: pd.DataFrame +) -> GoogleSheetValues: + mandatory_columns_error_message = "" + if not all( + col in new_campaign_df.columns for col in NEW_CAMPAIGN_MANDATORY_COLUMNS + ): + mandatory_columns_error_message = f"""Mandatory columns missing in the new campaign data. +Please provide the following columns: {NEW_CAMPAIGN_MANDATORY_COLUMNS}""" + + if not all(col in template_df.columns for col in MANDATORY_TEMPLATE_COLUMNS): + mandatory_columns_error_message = f"""Mandatory columns missing in the template data. +Please provide the following columns: {MANDATORY_TEMPLATE_COLUMNS}""" + if mandatory_columns_error_message: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=mandatory_columns_error_message, + ) + + # create new dataframe with columns from the template data + # For each row in the new campaign data frame, create two rows in the final data frame + # Both rows will have the same data except for the Ad Group column + # One will have "Station From - Station To" and the other will have "Station To - Station From" + # Campaign value will be the same for both rows - "Country - Station From - Station To" + final_df = pd.DataFrame(columns=template_df.columns) + for _, row in new_campaign_df.iterrows(): + campaign = f"{row['Country']} - {row['Station From']} - {row['Station To']}" + for ad_group in [ + f"{row['Station From']} - {row['Station To']}", + f"{row['Station To']} - {row['Station From']}", + ]: + new_row = row.copy() + new_row["Campaign"] = campaign + new_row["Ad Group"] = ad_group + final_df = final_df.append(new_row, ignore_index=True) + # AttributeError: 'DataFrame' object has no attribute 'append' + + return GoogleSheetValues(values=final_df.values.tolist()) + + +@app.post("/process-data") +async def process_data( + template_sheet_values: GoogleSheetValues, + new_campaign_sheet_values: GoogleSheetValues, + target_resource: Annotated[ + Literal["ad", "keyword"], Query(description="The target resource to be updated") + ], +) -> GoogleSheetValues: + if ( + len(template_sheet_values.values) < 2 + or len(new_campaign_sheet_values.values) < 2 + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Both template and new campaign data should have at least two rows (header and data).", + ) + try: + template_df = pd.DataFrame( + template_sheet_values.values[1:], columns=template_sheet_values.values[0] + ) + new_campaign_df = pd.DataFrame( + new_campaign_sheet_values.values[1:], + columns=new_campaign_sheet_values.values[0], + ) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid data format. Please provide data in the correct format: {e}", + ) from e + + if target_resource == "ad": + return process_ad_data(template_df, new_campaign_df) + + raise NotImplementedError("Processing for keyword data is not implemented yet.") diff --git a/pyproject.toml b/pyproject.toml index 2c9712d..4ff95d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ dependencies = [ "prisma==0.13.1", "google-api-python-client==2.133.0", "asyncify==0.10.0", + "pandas==2.2.2" ] [project.optional-dependencies] From 643e0d72a7593ab3e63274a75fdf52f5b8a82f80 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 27 Jun 2024 10:33:48 +0200 Subject: [PATCH 02/17] wip --- google_sheets/app.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index 8cec05f..39c03cf 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -300,11 +300,6 @@ def process_ad_data( detail=mandatory_columns_error_message, ) - # create new dataframe with columns from the template data - # For each row in the new campaign data frame, create two rows in the final data frame - # Both rows will have the same data except for the Ad Group column - # One will have "Station From - Station To" and the other will have "Station To - Station From" - # Campaign value will be the same for both rows - "Country - Station From - Station To" final_df = pd.DataFrame(columns=template_df.columns) for _, row in new_campaign_df.iterrows(): campaign = f"{row['Country']} - {row['Station From']} - {row['Station To']}" @@ -312,11 +307,10 @@ def process_ad_data( f"{row['Station From']} - {row['Station To']}", f"{row['Station To']} - {row['Station From']}", ]: - new_row = row.copy() + new_row = template_df.iloc[0].copy() new_row["Campaign"] = campaign new_row["Ad Group"] = ad_group - final_df = final_df.append(new_row, ignore_index=True) - # AttributeError: 'DataFrame' object has no attribute 'append' + final_df = pd.concat([final_df, pd.DataFrame([new_row])], ignore_index=True) return GoogleSheetValues(values=final_df.values.tolist()) From 5872d34526905be86ab426fce3e8976e060a7b24 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 27 Jun 2024 11:09:24 +0200 Subject: [PATCH 03/17] wip --- google_sheets/app.py | 63 +++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index 39c03cf..f831fae 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -269,7 +269,7 @@ async def get_all_sheet_titles( NEW_CAMPAIGN_MANDATORY_COLUMNS = ["Country", "Station From", "Station To"] -MANDATORY_TEMPLATE_COLUMNS = [ +MANDATORY_AD_TEMPLATE_COLUMNS = [ "Campaign", "Ad Group", "Headline 1", @@ -281,36 +281,31 @@ async def get_all_sheet_titles( ] +def validate_data(df: pd.DataFrame, mandatory_columns: List[str], name: str) -> str: + if not all(col in df.columns for col in mandatory_columns): + return f"""Mandatory columns missing in the {name} data. +Please provide the following columns: {mandatory_columns} +""" + return "" + + def process_ad_data( template_df: pd.DataFrame, new_campaign_df: pd.DataFrame ) -> GoogleSheetValues: - mandatory_columns_error_message = "" - if not all( - col in new_campaign_df.columns for col in NEW_CAMPAIGN_MANDATORY_COLUMNS - ): - mandatory_columns_error_message = f"""Mandatory columns missing in the new campaign data. -Please provide the following columns: {NEW_CAMPAIGN_MANDATORY_COLUMNS}""" - - if not all(col in template_df.columns for col in MANDATORY_TEMPLATE_COLUMNS): - mandatory_columns_error_message = f"""Mandatory columns missing in the template data. -Please provide the following columns: {MANDATORY_TEMPLATE_COLUMNS}""" - if mandatory_columns_error_message: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=mandatory_columns_error_message, - ) - final_df = pd.DataFrame(columns=template_df.columns) - for _, row in new_campaign_df.iterrows(): - campaign = f"{row['Country']} - {row['Station From']} - {row['Station To']}" - for ad_group in [ - f"{row['Station From']} - {row['Station To']}", - f"{row['Station To']} - {row['Station From']}", - ]: - new_row = template_df.iloc[0].copy() - new_row["Campaign"] = campaign - new_row["Ad Group"] = ad_group - final_df = pd.concat([final_df, pd.DataFrame([new_row])], ignore_index=True) + for _, template_row in template_df.iterrows(): + for _, new_campaign_row in new_campaign_df.iterrows(): + campaign = f"{new_campaign_row['Country']} - {new_campaign_row['Station From']} - {new_campaign_row['Station To']}" + for ad_group in [ + f"{new_campaign_row['Station From']} - {new_campaign_row['Station To']}", + f"{new_campaign_row['Station To']} - {new_campaign_row['Station From']}", + ]: + new_row = template_row.copy() + new_row["Campaign"] = campaign + new_row["Ad Group"] = ad_group + final_df = pd.concat( + [final_df, pd.DataFrame([new_row])], ignore_index=True + ) return GoogleSheetValues(values=final_df.values.tolist()) @@ -345,7 +340,21 @@ async def process_data( detail=f"Invalid data format. Please provide data in the correct format: {e}", ) from e + validation_error_msg = validate_data( + df=template_df, + mandatory_columns=MANDATORY_AD_TEMPLATE_COLUMNS, + name="ads template", + ) if target_resource == "ad": + validation_error_msg += validate_data( + df=new_campaign_df, + mandatory_columns=NEW_CAMPAIGN_MANDATORY_COLUMNS, + name="new campaign", + ) + if validation_error_msg: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_msg + ) return process_ad_data(template_df, new_campaign_df) raise NotImplementedError("Processing for keyword data is not implemented yet.") From 5031e0ba33d4c077360599c1d15fb2780076c1db Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 27 Jun 2024 11:35:34 +0200 Subject: [PATCH 04/17] process-data endpoint implemented --- google_sheets/app.py | 48 ++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index f831fae..baf3eb3 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -280,6 +280,14 @@ async def get_all_sheet_titles( "Final Url", ] +MANDATORY_KEYWORD_TEMPLATE_COLUMNS = [ + "Campaign", + "Ad Group", + "Keyword", + "Criterion Type", + "Max CPC", +] + def validate_data(df: pd.DataFrame, mandatory_columns: List[str], name: str) -> str: if not all(col in df.columns for col in mandatory_columns): @@ -289,7 +297,7 @@ def validate_data(df: pd.DataFrame, mandatory_columns: List[str], name: str) -> return "" -def process_ad_data( +def _process_data( template_df: pd.DataFrame, new_campaign_df: pd.DataFrame ) -> GoogleSheetValues: final_df = pd.DataFrame(columns=template_df.columns) @@ -307,10 +315,14 @@ def process_ad_data( [final_df, pd.DataFrame([new_row])], ignore_index=True ) - return GoogleSheetValues(values=final_df.values.tolist()) + values = [final_df.columns.tolist(), *final_df.values.tolist()] + return GoogleSheetValues(values=values) -@app.post("/process-data") +@app.post( + "/process-data", + description="Process data to generate new ads or keywords based on the template", +) async def process_data( template_sheet_values: GoogleSheetValues, new_campaign_sheet_values: GoogleSheetValues, @@ -341,20 +353,26 @@ async def process_data( ) from e validation_error_msg = validate_data( - df=template_df, - mandatory_columns=MANDATORY_AD_TEMPLATE_COLUMNS, - name="ads template", + df=new_campaign_df, + mandatory_columns=NEW_CAMPAIGN_MANDATORY_COLUMNS, + name="new campaign", ) + if target_resource == "ad": validation_error_msg += validate_data( - df=new_campaign_df, - mandatory_columns=NEW_CAMPAIGN_MANDATORY_COLUMNS, - name="new campaign", + df=template_df, + mandatory_columns=MANDATORY_AD_TEMPLATE_COLUMNS, + name="ads template", + ) + else: + validation_error_msg += validate_data( + df=template_df, + mandatory_columns=MANDATORY_KEYWORD_TEMPLATE_COLUMNS, + name="keyword template", + ) + if validation_error_msg: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_msg ) - if validation_error_msg: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_msg - ) - return process_ad_data(template_df, new_campaign_df) - raise NotImplementedError("Processing for keyword data is not implemented yet.") + return _process_data(template_df, new_campaign_df) From b9607e1a6412ce419f4da8f6bb31350c86b10ba0 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 27 Jun 2024 11:49:04 +0200 Subject: [PATCH 05/17] Updated get-sheet endpoint to return pydantic GoogleSheetValues object --- google_sheets/app.py | 44 +++++++++++++++++++++++++++++++++++++++++-- tests/app/test_app.py | 7 +++++-- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index baf3eb3..2001940 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -136,7 +136,7 @@ async def get_sheet( str, Query(description="The title of the sheet to fetch data from"), ], -) -> Union[str, List[List[str]]]: +) -> Union[str, GoogleSheetValues]: service = await build_service(user_id=user_id, service_name="sheets", version="v4") values = await get_sheet_f( service=service, spreadsheet_id=spreadsheet_id, range=title @@ -145,7 +145,7 @@ async def get_sheet( if not values: return "No data found." - return values # type: ignore[no-any-return] + return GoogleSheetValues(values=values) @app.post( @@ -376,3 +376,43 @@ async def process_data( ) return _process_data(template_df, new_campaign_df) + + +# process-spreadsheet endpoint +# input: user_id, template_spreadsheet_id, template_sheet_title, new_campaign_spreadsheet_id, new_campaign_sheet_title, target_resource + + +# output: new sheet within the new_campaign_spreadsheet_id with the processed data and 201 status code +@app.post( + "/process-spreadsheet", + description="Process data to generate new ads or keywords based on the template", +) +async def process_spreadsheet( + user_id: Annotated[ + int, Query(description="The user ID for which the data is requested") + ], + template_spreadsheet_id: Annotated[ + str, Query(description="ID of the Google Sheet with the template data") + ], + template_sheet_title: Annotated[ + str, + Query(description="The title of the sheet with the template data"), + ], + new_campaign_spreadsheet_id: Annotated[ + str, Query(description="ID of the Google Sheet with the new campaign data") + ], + new_campaign_sheet_title: Annotated[ + str, + Query(description="The title of the sheet with the new campaign data"), + ], + target_resource: Annotated[ + Literal["ad", "keyword"], Query(description="The target resource to be updated") + ], +) -> Response: + # service = await build_service(user_id=user_id, service_name="sheets", version="v4") + + # try: + # template_values = await get_sheet(user_id=user_id, spreadsheet_id=template_spreadsheet_id, title=template_sheet_title) + # new_campaign_values = await get_sheet(user_id=user_id, spreadsheet_id=new_campaign_spreadsheet_id, title=new_campaign_sheet_title) + + raise NotImplementedError("This endpoint is not implemented yet.") diff --git a/tests/app/test_app.py b/tests/app/test_app.py index 96d982c..2861207 100644 --- a/tests/app/test_app.py +++ b/tests/app/test_app.py @@ -7,6 +7,7 @@ from google_sheets import __version__ as version from google_sheets.app import app +from google_sheets.model import GoogleSheetValues client = TestClient(app) @@ -17,14 +18,14 @@ def test_get_sheet(self) -> None: "google_sheets.google_api.service._load_user_credentials", return_value={"refresh_token": "abcdf"}, ) as mock_load_user_credentials: - excepted = [ + values = [ ["Campaign", "Ad Group", "Keyword"], ["Campaign A", "Ad group A", "Keyword A"], ["Campaign A", "Ad group A", "Keyword B"], ["Campaign A", "Ad group A", "Keyword C"], ] with patch( - "google_sheets.app.get_sheet_f", return_value=excepted + "google_sheets.app.get_sheet_f", return_value=values ) as mock_get_sheet: response = client.get( "/get-sheet?user_id=123&spreadsheet_id=abc&title=Sheet1" @@ -32,6 +33,8 @@ def test_get_sheet(self) -> None: mock_load_user_credentials.assert_called_once() mock_get_sheet.assert_called_once() assert response.status_code == 200 + + excepted = GoogleSheetValues(values=values).model_dump() assert response.json() == excepted From c32196d57137b2d6d311e9b0a75962f20f029bad Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 27 Jun 2024 12:47:23 +0200 Subject: [PATCH 06/17] process-spreadsheet endpoint implemented --- google_sheets/app.py | 67 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index 2001940..2bca0f9 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -1,5 +1,6 @@ import json import logging +from datetime import datetime from os import environ from typing import Annotated, Dict, List, Literal, Union @@ -290,10 +291,17 @@ async def get_all_sheet_titles( def validate_data(df: pd.DataFrame, mandatory_columns: List[str], name: str) -> str: + error_msg = "" + if len(df.columns) != len(set(df.columns)): + error_msg = f"""Duplicate columns found in the {name} data. +Please provide unique column names. +""" if not all(col in df.columns for col in mandatory_columns): - return f"""Mandatory columns missing in the {name} data. + error_msg += f"""Mandatory columns missing in the {name} data. Please provide the following columns: {mandatory_columns} """ + if error_msg: + return error_msg return "" @@ -378,11 +386,6 @@ async def process_data( return _process_data(template_df, new_campaign_df) -# process-spreadsheet endpoint -# input: user_id, template_spreadsheet_id, template_sheet_title, new_campaign_spreadsheet_id, new_campaign_sheet_title, target_resource - - -# output: new sheet within the new_campaign_spreadsheet_id with the processed data and 201 status code @app.post( "/process-spreadsheet", description="Process data to generate new ads or keywords based on the template", @@ -409,10 +412,52 @@ async def process_spreadsheet( Literal["ad", "keyword"], Query(description="The target resource to be updated") ], ) -> Response: - # service = await build_service(user_id=user_id, service_name="sheets", version="v4") + template_values = await get_sheet( + user_id=user_id, + spreadsheet_id=template_spreadsheet_id, + title=template_sheet_title, + ) + new_campaign_values = await get_sheet( + user_id=user_id, + spreadsheet_id=new_campaign_spreadsheet_id, + title=new_campaign_sheet_title, + ) + + if not isinstance(template_values, GoogleSheetValues) or not isinstance( + new_campaign_values, GoogleSheetValues + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"""Invalid data format. +template_values: {template_values} + +new_campaign_values: {new_campaign_values} - # try: - # template_values = await get_sheet(user_id=user_id, spreadsheet_id=template_spreadsheet_id, title=template_sheet_title) - # new_campaign_values = await get_sheet(user_id=user_id, spreadsheet_id=new_campaign_spreadsheet_id, title=new_campaign_sheet_title) +Please provide data in the correct format.""", + ) + + processed_values = await process_data( + template_sheet_values=template_values, + new_campaign_sheet_values=new_campaign_values, + target_resource=target_resource, + ) + + title = ( + f"Captn - {target_resource.capitalize()}s {datetime.now():%Y-%m-%d %H:%M:%S}" + ) + await create_sheet( + user_id=user_id, + spreadsheet_id=new_campaign_spreadsheet_id, + title=title, + ) + await update_sheet( + user_id=user_id, + spreadsheet_id=new_campaign_spreadsheet_id, + title=title, + sheet_values=processed_values, + ) - raise NotImplementedError("This endpoint is not implemented yet.") + return Response( + status_code=status.HTTP_201_CREATED, + content=f"Sheet with the name 'Captn - {target_resource.capitalize()}s' has been created successfully.", + ) From 256500decdbd16a260de5f765519166d9ba864bb Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 27 Jun 2024 12:56:50 +0200 Subject: [PATCH 07/17] Refactoring --- google_sheets/app.py | 40 +------------------ google_sheets/data_processing/__init__.py | 3 ++ google_sheets/data_processing/processing.py | 44 +++++++++++++++++++++ 3 files changed, 49 insertions(+), 38 deletions(-) create mode 100644 google_sheets/data_processing/__init__.py create mode 100644 google_sheets/data_processing/processing.py diff --git a/google_sheets/app.py b/google_sheets/app.py index 2bca0f9..64696ae 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -11,6 +11,7 @@ from googleapiclient.errors import HttpError from . import __version__ +from .data_processing import process_data_f, validate_data from .db_helpers import get_db_connection from .google_api import ( build_service, @@ -290,43 +291,6 @@ async def get_all_sheet_titles( ] -def validate_data(df: pd.DataFrame, mandatory_columns: List[str], name: str) -> str: - error_msg = "" - if len(df.columns) != len(set(df.columns)): - error_msg = f"""Duplicate columns found in the {name} data. -Please provide unique column names. -""" - if not all(col in df.columns for col in mandatory_columns): - error_msg += f"""Mandatory columns missing in the {name} data. -Please provide the following columns: {mandatory_columns} -""" - if error_msg: - return error_msg - return "" - - -def _process_data( - template_df: pd.DataFrame, new_campaign_df: pd.DataFrame -) -> GoogleSheetValues: - final_df = pd.DataFrame(columns=template_df.columns) - for _, template_row in template_df.iterrows(): - for _, new_campaign_row in new_campaign_df.iterrows(): - campaign = f"{new_campaign_row['Country']} - {new_campaign_row['Station From']} - {new_campaign_row['Station To']}" - for ad_group in [ - f"{new_campaign_row['Station From']} - {new_campaign_row['Station To']}", - f"{new_campaign_row['Station To']} - {new_campaign_row['Station From']}", - ]: - new_row = template_row.copy() - new_row["Campaign"] = campaign - new_row["Ad Group"] = ad_group - final_df = pd.concat( - [final_df, pd.DataFrame([new_row])], ignore_index=True - ) - - values = [final_df.columns.tolist(), *final_df.values.tolist()] - return GoogleSheetValues(values=values) - - @app.post( "/process-data", description="Process data to generate new ads or keywords based on the template", @@ -383,7 +347,7 @@ async def process_data( status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_msg ) - return _process_data(template_df, new_campaign_df) + return process_data_f(template_df, new_campaign_df) @app.post( diff --git a/google_sheets/data_processing/__init__.py b/google_sheets/data_processing/__init__.py new file mode 100644 index 0000000..3718adf --- /dev/null +++ b/google_sheets/data_processing/__init__.py @@ -0,0 +1,3 @@ +from .processing import process_data_f, validate_data + +__all__ = ["process_data_f", "validate_data"] diff --git a/google_sheets/data_processing/processing.py b/google_sheets/data_processing/processing.py new file mode 100644 index 0000000..547411d --- /dev/null +++ b/google_sheets/data_processing/processing.py @@ -0,0 +1,44 @@ +from typing import List + +import pandas as pd + +from ..model import GoogleSheetValues + +__all__ = ["process_data_f", "validate_data"] + + +def validate_data(df: pd.DataFrame, mandatory_columns: List[str], name: str) -> str: + error_msg = "" + if len(df.columns) != len(set(df.columns)): + error_msg = f"""Duplicate columns found in the {name} data. +Please provide unique column names. +""" + if not all(col in df.columns for col in mandatory_columns): + error_msg += f"""Mandatory columns missing in the {name} data. +Please provide the following columns: {mandatory_columns} +""" + if error_msg: + return error_msg + return "" + + +def process_data_f( + template_df: pd.DataFrame, new_campaign_df: pd.DataFrame +) -> GoogleSheetValues: + final_df = pd.DataFrame(columns=template_df.columns) + for _, template_row in template_df.iterrows(): + for _, new_campaign_row in new_campaign_df.iterrows(): + campaign = f"{new_campaign_row['Country']} - {new_campaign_row['Station From']} - {new_campaign_row['Station To']}" + for ad_group in [ + f"{new_campaign_row['Station From']} - {new_campaign_row['Station To']}", + f"{new_campaign_row['Station To']} - {new_campaign_row['Station From']}", + ]: + new_row = template_row.copy() + new_row["Campaign"] = campaign + new_row["Ad Group"] = ad_group + final_df = pd.concat( + [final_df, pd.DataFrame([new_row])], ignore_index=True + ) + + values = [final_df.columns.tolist(), *final_df.values.tolist()] + return GoogleSheetValues(values=values) From 81cc9ea3e72906fd668bf2c11d8d260df68fe28c Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Thu, 27 Jun 2024 21:05:38 +0200 Subject: [PATCH 08/17] Tests updated --- tests/app/fixtures/openapi.json | 1 + tests/app/test_app.py | 575 +++++------------------ tests/data_processing/test_processing.py | 75 +++ 3 files changed, 186 insertions(+), 465 deletions(-) create mode 100644 tests/app/fixtures/openapi.json create mode 100644 tests/data_processing/test_processing.py diff --git a/tests/app/fixtures/openapi.json b/tests/app/fixtures/openapi.json new file mode 100644 index 0000000..dae4ed4 --- /dev/null +++ b/tests/app/fixtures/openapi.json @@ -0,0 +1 @@ +{"openapi":"3.1.0","info":{"title":"google-sheets","version":"0.1.0"},"servers":[{"url":"http://localhost:8000","description":"Google Sheets app server"}],"paths":{"/login":{"get":{"summary":"Get Login Url","operationId":"get_login_url_login_get","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","title":"User ID"}},{"name":"force_new_login","in":"query","required":false,"schema":{"type":"boolean","title":"Force new login","default":false}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"type":"object","additionalProperties":{"type":"string"},"title":"Response Get Login Url Login Get"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/login/success":{"get":{"summary":"Get Login Success","operationId":"get_login_success_login_success_get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"additionalProperties":{"type":"string"},"type":"object","title":"Response Get Login Success Login Success Get"}}}}}}},"/login/callback":{"get":{"summary":"Login Callback","operationId":"login_callback_login_callback_get","parameters":[{"name":"code","in":"query","required":true,"schema":{"type":"string","title":"Authorization Code"}},{"name":"state","in":"query","required":true,"schema":{"type":"string","title":"State"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/get-sheet":{"get":{"summary":"Get Sheet","description":"Get data from a Google Sheet","operationId":"get_sheet_get_sheet_get","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","description":"The user ID for which the data is requested","title":"User Id"},"description":"The user ID for which the data is requested"},{"name":"spreadsheet_id","in":"query","required":true,"schema":{"type":"string","description":"ID of the Google Sheet to fetch data from","title":"Spreadsheet Id"},"description":"ID of the Google Sheet to fetch data from"},{"name":"title","in":"query","required":true,"schema":{"type":"string","description":"The title of the sheet to fetch data from","title":"Title"},"description":"The title of the sheet to fetch data from"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"anyOf":[{"type":"string"},{"$ref":"#/components/schemas/GoogleSheetValues"}],"title":"Response Get Sheet Get Sheet Get"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/update-sheet":{"post":{"summary":"Update Sheet","description":"Update data in a Google Sheet within the existing spreadsheet","operationId":"update_sheet_update_sheet_post","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","description":"The user ID for which the data is requested","title":"User Id"},"description":"The user ID for which the data is requested"},{"name":"spreadsheet_id","in":"query","required":true,"schema":{"type":"string","description":"ID of the Google Sheet to fetch data from","title":"Spreadsheet Id"},"description":"ID of the Google Sheet to fetch data from"},{"name":"title","in":"query","required":true,"schema":{"type":"string","description":"The title of the sheet to update","title":"Title"},"description":"The title of the sheet to update"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GoogleSheetValues"}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/create-sheet":{"post":{"summary":"Create Sheet","description":"Create a new Google Sheet within the existing spreadsheet","operationId":"create_sheet_create_sheet_post","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","description":"The user ID for which the data is requested","title":"User Id"},"description":"The user ID for which the data is requested"},{"name":"spreadsheet_id","in":"query","required":true,"schema":{"type":"string","description":"ID of the Google Sheet to fetch data from","title":"Spreadsheet Id"},"description":"ID of the Google Sheet to fetch data from"},{"name":"title","in":"query","required":true,"schema":{"type":"string","description":"The title of the new sheet","title":"Title"},"description":"The title of the new sheet"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/get-all-file-names":{"get":{"summary":"Get All File Names","description":"Get all sheets associated with the user","operationId":"get_all_file_names_get_all_file_names_get","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","description":"The user ID for which the data is requested","title":"User Id"},"description":"The user ID for which the data is requested"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"type":"object","additionalProperties":{"type":"string"},"title":"Response Get All File Names Get All File Names Get"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/get-all-sheet-titles":{"get":{"summary":"Get All Sheet Titles","description":"Get all sheet titles within a Google Spreadsheet","operationId":"get_all_sheet_titles_get_all_sheet_titles_get","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","description":"The user ID for which the data is requested","title":"User Id"},"description":"The user ID for which the data is requested"},{"name":"spreadsheet_id","in":"query","required":true,"schema":{"type":"string","description":"ID of the Google Sheet to fetch data from","title":"Spreadsheet Id"},"description":"ID of the Google Sheet to fetch data from"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"type":"array","items":{"type":"string"},"title":"Response Get All Sheet Titles Get All Sheet Titles Get"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/process-data":{"post":{"summary":"Process Data","description":"Process data to generate new ads or keywords based on the template","operationId":"process_data_process_data_post","parameters":[{"name":"target_resource","in":"query","required":true,"schema":{"enum":["ad","keyword"],"type":"string","description":"The target resource to be updated","title":"Target Resource"},"description":"The target resource to be updated"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Body_process_data_process_data_post"}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GoogleSheetValues"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/process-spreadsheet":{"post":{"summary":"Process Spreadsheet","description":"Process data to generate new ads or keywords based on the template","operationId":"process_spreadsheet_process_spreadsheet_post","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","description":"The user ID for which the data is requested","title":"User Id"},"description":"The user ID for which the data is requested"},{"name":"template_spreadsheet_id","in":"query","required":true,"schema":{"type":"string","description":"ID of the Google Sheet with the template data","title":"Template Spreadsheet Id"},"description":"ID of the Google Sheet with the template data"},{"name":"template_sheet_title","in":"query","required":true,"schema":{"type":"string","description":"The title of the sheet with the template data","title":"Template Sheet Title"},"description":"The title of the sheet with the template data"},{"name":"new_campaign_spreadsheet_id","in":"query","required":true,"schema":{"type":"string","description":"ID of the Google Sheet with the new campaign data","title":"New Campaign Spreadsheet Id"},"description":"ID of the Google Sheet with the new campaign data"},{"name":"new_campaign_sheet_title","in":"query","required":true,"schema":{"type":"string","description":"The title of the sheet with the new campaign data","title":"New Campaign Sheet Title"},"description":"The title of the sheet with the new campaign data"},{"name":"target_resource","in":"query","required":true,"schema":{"enum":["ad","keyword"],"type":"string","description":"The target resource to be updated","title":"Target Resource"},"description":"The target resource to be updated"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}}},"components":{"schemas":{"Body_process_data_process_data_post":{"properties":{"template_sheet_values":{"$ref":"#/components/schemas/GoogleSheetValues"},"new_campaign_sheet_values":{"$ref":"#/components/schemas/GoogleSheetValues"}},"type":"object","required":["template_sheet_values","new_campaign_sheet_values"],"title":"Body_process_data_process_data_post"},"GoogleSheetValues":{"properties":{"values":{"items":{"items":{},"type":"array"},"type":"array","title":"Values","description":"Values to be written to the Google Sheet."}},"type":"object","required":["values"],"title":"GoogleSheetValues"},"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"}}}} diff --git a/tests/app/test_app.py b/tests/app/test_app.py index 2861207..90f0de1 100644 --- a/tests/app/test_app.py +++ b/tests/app/test_app.py @@ -1,3 +1,5 @@ +import json +from pathlib import Path from typing import Optional, Union from unittest.mock import MagicMock, patch @@ -5,7 +7,6 @@ from fastapi.testclient import TestClient from googleapiclient.errors import HttpError -from google_sheets import __version__ as version from google_sheets.app import app from google_sheets.model import GoogleSheetValues @@ -163,474 +164,118 @@ def test_get_all_file_names(self) -> None: assert response.json() == expected -class TestOpenAPIJSON: - def test_openapi(self) -> None: - expected = { - "openapi": "3.1.0", - "info": {"title": "google-sheets", "version": version}, - "servers": [ - { - "url": "http://localhost:8000", - "description": "Google Sheets app server", - } - ], - "paths": { - "/login": { - "get": { - "summary": "Get Login Url", - "operationId": "get_login_url_login_get", - "parameters": [ - { - "name": "user_id", - "in": "query", - "required": True, - "schema": {"type": "integer", "title": "User ID"}, - }, - { - "name": "force_new_login", - "in": "query", - "required": False, - "schema": { - "type": "boolean", - "title": "Force new login", - "default": False, - }, - }, - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": {"type": "string"}, - "title": "Response Get Login Url Login Get", - } - } - }, - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - }, - }, - } - }, - "/login/success": { - "get": { - "summary": "Get Login Success", - "operationId": "get_login_success_login_success_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": {"type": "string"}, - "type": "object", - "title": "Response Get Login Success Login Success Get", - } - } - }, - } - }, - } - }, - "/login/callback": { - "get": { - "summary": "Login Callback", - "operationId": "login_callback_login_callback_get", - "parameters": [ - { - "name": "code", - "in": "query", - "required": True, - "schema": { - "type": "string", - "title": "Authorization Code", - }, - }, - { - "name": "state", - "in": "query", - "required": True, - "schema": {"type": "string", "title": "State"}, - }, - ], - "responses": { - "200": { - "description": "Successful Response", - "content": {"application/json": {"schema": {}}}, - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - }, - }, - } - }, - "/get-sheet": { - "get": { - "summary": "Get Sheet", - "description": "Get data from a Google Sheet", - "operationId": "get_sheet_get_sheet_get", - "parameters": [ - { - "name": "user_id", - "in": "query", - "required": True, - "schema": { - "type": "integer", - "description": "The user ID for which the data is requested", - "title": "User Id", - }, - "description": "The user ID for which the data is requested", - }, - { - "name": "spreadsheet_id", - "in": "query", - "required": True, - "schema": { - "type": "string", - "description": "ID of the Google Sheet to fetch data from", - "title": "Spreadsheet Id", - }, - "description": "ID of the Google Sheet to fetch data from", - }, - { - "name": "title", - "in": "query", - "required": True, - "schema": { - "type": "string", - "description": "The title of the sheet to fetch data from", - "title": "Title", - }, - "description": "The title of the sheet to fetch data from", - }, - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - {"type": "string"}, - { - "type": "array", - "items": { - "type": "array", - "items": {"type": "string"}, - }, - }, - ], - "title": "Response Get Sheet Get Sheet Get", - } - } - }, - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - }, - }, - } - }, - "/update-sheet": { - "post": { - "summary": "Update Sheet", - "description": "Update data in a Google Sheet within the existing spreadsheet", - "operationId": "update_sheet_update_sheet_post", - "parameters": [ - { - "name": "user_id", - "in": "query", - "required": True, - "schema": { - "type": "integer", - "description": "The user ID for which the data is requested", - "title": "User Id", - }, - "description": "The user ID for which the data is requested", - }, - { - "name": "spreadsheet_id", - "in": "query", - "required": True, - "schema": { - "type": "string", - "description": "ID of the Google Sheet to fetch data from", - "title": "Spreadsheet Id", - }, - "description": "ID of the Google Sheet to fetch data from", - }, - { - "name": "title", - "in": "query", - "required": True, - "schema": { - "type": "string", - "description": "The title of the sheet to update", - "title": "Title", - }, - "description": "The title of the sheet to update", - }, +class TestProcessData: + @pytest.mark.parametrize( + ("template_sheet_values", "new_campaign_sheet_values", "status_code", "detail"), + [ + ( + GoogleSheetValues( + values=[ + ["Campaign", "Ad Group", "Keyword"], + ] + ), + GoogleSheetValues( + values=[ + ["Country", "Station From", "Station To"], + ["India", "Delhi", "Mumbai"], + ] + ), + 400, + "Both template and new campaign data should have at least two rows", + ), + ( + GoogleSheetValues( + values=[ + ["Campaign", "Ad Group", "Keyword"], + ["Campaign A", "Ad group A", "Keyword A"], + ] + ), + GoogleSheetValues( + values=[ + ["Country", "Station From", "Station To"], + ["India", "Delhi", "Mumbai"], + ] + ), + 400, + "Mandatory columns missing in the keyword template data.", + ), + ( + GoogleSheetValues( + values=[ + [ + "Campaign", + "Ad Group", + "Keyword", + "Criterion Type", + "Max CPC", ], - "requestBody": { - "required": True, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/GoogleSheetValues" - } - } - }, - }, - "responses": { - "200": { - "description": "Successful Response", - "content": {"application/json": {"schema": {}}}, - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - }, - }, - } - }, - "/create-sheet": { - "post": { - "summary": "Create Sheet", - "description": "Create a new Google Sheet within the existing spreadsheet", - "operationId": "create_sheet_create_sheet_post", - "parameters": [ - { - "name": "user_id", - "in": "query", - "required": True, - "schema": { - "type": "integer", - "description": "The user ID for which the data is requested", - "title": "User Id", - }, - "description": "The user ID for which the data is requested", - }, - { - "name": "spreadsheet_id", - "in": "query", - "required": True, - "schema": { - "type": "string", - "description": "ID of the Google Sheet to fetch data from", - "title": "Spreadsheet Id", - }, - "description": "ID of the Google Sheet to fetch data from", - }, - { - "name": "title", - "in": "query", - "required": True, - "schema": { - "type": "string", - "description": "The title of the new sheet", - "title": "Title", - }, - "description": "The title of the new sheet", - }, + ["Campaign A", "Ad group A", "Keyword A", "Exact", "1"], + ] + ), + GoogleSheetValues( + values=[ + ["Country", "Station From", "Station To"], + ["India", "Delhi", "Mumbai"], + ] + ), + 200, + GoogleSheetValues( + values=[ + [ + "Campaign", + "Ad Group", + "Keyword", + "Criterion Type", + "Max CPC", ], - "responses": { - "200": { - "description": "Successful Response", - "content": {"application/json": {"schema": {}}}, - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - }, - }, - } - }, - "/get-all-file-names": { - "get": { - "summary": "Get All File Names", - "description": "Get all sheets associated with the user", - "operationId": "get_all_file_names_get_all_file_names_get", - "parameters": [ - { - "name": "user_id", - "in": "query", - "required": True, - "schema": { - "type": "integer", - "description": "The user ID for which the data is requested", - "title": "User Id", - }, - "description": "The user ID for which the data is requested", - } + [ + "India - Delhi - Mumbai", + "Delhi - Mumbai", + "Keyword A", + "Exact", + "1", ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": {"type": "string"}, - "title": "Response Get All File Names Get All File Names Get", - } - } - }, - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - }, - }, - } - }, - "/get-all-sheet-titles": { - "get": { - "summary": "Get All Sheet Titles", - "description": "Get all sheet titles within a Google Spreadsheet", - "operationId": "get_all_sheet_titles_get_all_sheet_titles_get", - "parameters": [ - { - "name": "user_id", - "in": "query", - "required": True, - "schema": { - "type": "integer", - "description": "The user ID for which the data is requested", - "title": "User Id", - }, - "description": "The user ID for which the data is requested", - }, - { - "name": "spreadsheet_id", - "in": "query", - "required": True, - "schema": { - "type": "string", - "description": "ID of the Google Sheet to fetch data from", - "title": "Spreadsheet Id", - }, - "description": "ID of the Google Sheet to fetch data from", - }, + [ + "India - Delhi - Mumbai", + "Mumbai - Delhi", + "Keyword A", + "Exact", + "1", ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": {"type": "string"}, - "title": "Response Get All Sheet Titles Get All Sheet Titles Get", - } - } - }, - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - }, - }, - } - }, - }, - "components": { - "schemas": { - "GoogleSheetValues": { - "properties": { - "values": { - "items": {"items": {}, "type": "array"}, - "type": "array", - "title": "Values", - "description": "Values to be written to the Google Sheet.", - } - }, - "type": "object", - "required": ["values"], - "title": "GoogleSheetValues", - }, - "HTTPValidationError": { - "properties": { - "detail": { - "items": { - "$ref": "#/components/schemas/ValidationError" - }, - "type": "array", - "title": "Detail", - } - }, - "type": "object", - "title": "HTTPValidationError", - }, - "ValidationError": { - "properties": { - "loc": { - "items": { - "anyOf": [{"type": "string"}, {"type": "integer"}] - }, - "type": "array", - "title": "Location", - }, - "msg": {"type": "string", "title": "Message"}, - "type": {"type": "string", "title": "Error Type"}, - }, - "type": "object", - "required": ["loc", "msg", "type"], - "title": "ValidationError", - }, - } + ], + ), + ), + ], + ) + def test_process_data( + self, + template_sheet_values: GoogleSheetValues, + new_campaign_sheet_values: GoogleSheetValues, + status_code: int, + detail: Union[str, GoogleSheetValues], + ) -> None: + response = client.post( + "/process-data?target_resource=keyword", + json={ + "template_sheet_values": template_sheet_values.model_dump(), + "new_campaign_sheet_values": new_campaign_sheet_values.model_dump(), }, - } + ) + + assert response.status_code == status_code + if isinstance(detail, GoogleSheetValues): + assert response.json() == detail.model_dump() + else: + assert detail in response.json()["detail"] + + +class TestOpenAPIJSON: + def test_openapi(self) -> None: + path = Path(__file__).parent / "fixtures" / "openapi.json" + with Path.open(path, "r") as f: + expected = f.read() + + expected_json = json.loads(expected) response = client.get("/openapi.json") assert response.status_code == 200 - resp_json = response.json() - - assert resp_json == expected + assert response.json() == expected_json diff --git a/tests/data_processing/test_processing.py b/tests/data_processing/test_processing.py new file mode 100644 index 0000000..7a55e10 --- /dev/null +++ b/tests/data_processing/test_processing.py @@ -0,0 +1,75 @@ +import pandas as pd +import pytest + +from google_sheets.data_processing.processing import process_data_f, validate_data + + +@pytest.mark.parametrize( + ("df", "expected"), + [ + ( + pd.DataFrame( + { + "Country": ["USA", "USA"], + "Station From": ["A", "B"], + "Station To": ["B", "A"], + } + ), + "", + ), + ( + pd.DataFrame( + { + "Country": ["USA", "USA"], + "Station From": ["A", "B"], + } + ), + """Mandatory columns missing in the name data. +Please provide the following columns: ['Country', 'Station From', 'Station To'] +""", + ), + ( + pd.DataFrame( + [["USA", "A", "B", "B"], ["USA", "B", "A", "C"]], + columns=["Country", "Station From", "Station To", "Station To"], + ), + """Duplicate columns found in the name data. +Please provide unique column names. +""", + ), + ], +) +def test_validate_data(df: pd.DataFrame, expected: str) -> None: + mandatory_columns = ["Country", "Station From", "Station To"] + assert validate_data(df, mandatory_columns, "name") == expected + + +def test_process_data_f() -> None: + template_df = pd.DataFrame( + { + "Campaign": ["", ""], + "Ad Group": ["", ""], + "Keyword": ["k1", "k2"], + "Max CPC": ["", ""], + } + ) + new_campaign_df = pd.DataFrame( + { + "Country": ["USA", "USA"], + "Station From": ["A", "B"], + "Station To": ["C", "D"], + } + ) + + expected = [ + ["Campaign", "Ad Group", "Keyword", "Max CPC"], + ["USA - A - C", "A - C", "k1", ""], + ["USA - A - C", "C - A", "k1", ""], + ["USA - B - D", "B - D", "k1", ""], + ["USA - B - D", "D - B", "k1", ""], + ["USA - A - C", "A - C", "k2", ""], + ["USA - A - C", "C - A", "k2", ""], + ["USA - B - D", "B - D", "k2", ""], + ["USA - B - D", "D - B", "k2", ""], + ] + assert process_data_f(template_df, new_campaign_df).values == expected From 96cc019b4bb17fc5a48e9634c8d8cff4e2a8aaa1 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 28 Jun 2024 11:14:17 +0200 Subject: [PATCH 09/17] wip --- google_sheets/data_processing/processing.py | 31 +++++-- tests/data_processing/test_processing.py | 94 +++++++++++++++------ 2 files changed, 92 insertions(+), 33 deletions(-) diff --git a/google_sheets/data_processing/processing.py b/google_sheets/data_processing/processing.py index 547411d..e2fc895 100644 --- a/google_sheets/data_processing/processing.py +++ b/google_sheets/data_processing/processing.py @@ -22,6 +22,10 @@ def validate_data(df: pd.DataFrame, mandatory_columns: List[str], name: str) -> return "" +INSERT_STATION_FROM = "INSERT_STATION_FROM" +INSERT_STATION_TO = "INSERT_STATION_TO" + + def process_data_f( template_df: pd.DataFrame, new_campaign_df: pd.DataFrame ) -> GoogleSheetValues: @@ -29,13 +33,30 @@ def process_data_f( for _, template_row in template_df.iterrows(): for _, new_campaign_row in new_campaign_df.iterrows(): campaign = f"{new_campaign_row['Country']} - {new_campaign_row['Station From']} - {new_campaign_row['Station To']}" - for ad_group in [ - f"{new_campaign_row['Station From']} - {new_campaign_row['Station To']}", - f"{new_campaign_row['Station To']} - {new_campaign_row['Station From']}", - ]: + stations = [ + { + "Station From": new_campaign_row["Station From"], + "Station To": new_campaign_row["Station To"], + }, + # Reverse the order of the stations + { + "Station From": new_campaign_row["Station To"], + "Station To": new_campaign_row["Station From"], + }, + ] + for station in stations: new_row = template_row.copy() new_row["Campaign"] = campaign - new_row["Ad Group"] = ad_group + new_row["Ad Group"] = ( + f"{station['Station From']} - {station['Station To']}" + ) + + # Replace the placeholders in all columns with the actual station names INSERT_STATION_FROM + new_row = new_row.str.replace( + INSERT_STATION_FROM, station["Station From"] + ) + new_row = new_row.str.replace(INSERT_STATION_TO, station["Station To"]) + final_df = pd.concat( [final_df, pd.DataFrame([new_row])], ignore_index=True ) diff --git a/tests/data_processing/test_processing.py b/tests/data_processing/test_processing.py index 7a55e10..b608e35 100644 --- a/tests/data_processing/test_processing.py +++ b/tests/data_processing/test_processing.py @@ -1,3 +1,5 @@ +from typing import List + import pandas as pd import pytest @@ -44,32 +46,68 @@ def test_validate_data(df: pd.DataFrame, expected: str) -> None: assert validate_data(df, mandatory_columns, "name") == expected -def test_process_data_f() -> None: - template_df = pd.DataFrame( - { - "Campaign": ["", ""], - "Ad Group": ["", ""], - "Keyword": ["k1", "k2"], - "Max CPC": ["", ""], - } - ) - new_campaign_df = pd.DataFrame( - { - "Country": ["USA", "USA"], - "Station From": ["A", "B"], - "Station To": ["C", "D"], - } - ) - - expected = [ - ["Campaign", "Ad Group", "Keyword", "Max CPC"], - ["USA - A - C", "A - C", "k1", ""], - ["USA - A - C", "C - A", "k1", ""], - ["USA - B - D", "B - D", "k1", ""], - ["USA - B - D", "D - B", "k1", ""], - ["USA - A - C", "A - C", "k2", ""], - ["USA - A - C", "C - A", "k2", ""], - ["USA - B - D", "B - D", "k2", ""], - ["USA - B - D", "D - B", "k2", ""], - ] +@pytest.mark.parametrize( + ("template_df", "new_campaign_df", "expected"), + [ + ( + pd.DataFrame( + { + "Campaign": ["", ""], + "Ad Group": ["", ""], + "Keyword": ["k1", "k2"], + "Max CPC": ["", ""], + } + ), + pd.DataFrame( + { + "Country": ["USA", "USA"], + "Station From": ["A", "B"], + "Station To": ["C", "D"], + } + ), + [ + ["Campaign", "Ad Group", "Keyword", "Max CPC"], + ["USA - A - C", "A - C", "k1", ""], + ["USA - A - C", "C - A", "k1", ""], + ["USA - B - D", "B - D", "k1", ""], + ["USA - B - D", "D - B", "k1", ""], + ["USA - A - C", "A - C", "k2", ""], + ["USA - A - C", "C - A", "k2", ""], + ["USA - B - D", "B - D", "k2", ""], + ["USA - B - D", "D - B", "k2", ""], + ], + ), + ( + pd.DataFrame( + { + "Campaign": ["", ""], + "Ad Group": ["", ""], + "Keyword": ["k1 INSERT_STATION_FROM", "k2"], + "Max CPC": ["", ""], + } + ), + pd.DataFrame( + { + "Country": ["USA", "USA"], + "Station From": ["A", "B"], + "Station To": ["C", "D"], + } + ), + [ + ["Campaign", "Ad Group", "Keyword", "Max CPC"], + ["USA - A - C", "A - C", "k1 A", ""], + ["USA - A - C", "C - A", "k1 C", ""], + ["USA - B - D", "B - D", "k1 B", ""], + ["USA - B - D", "D - B", "k1 D", ""], + ["USA - A - C", "A - C", "k2", ""], + ["USA - A - C", "C - A", "k2", ""], + ["USA - B - D", "B - D", "k2", ""], + ["USA - B - D", "D - B", "k2", ""], + ], + ), + ], +) +def test_process_data_f( + template_df: pd.DataFrame, new_campaign_df: pd.DataFrame, expected: List[List[str]] +) -> None: assert process_data_f(template_df, new_campaign_df).values == expected From f7efe510db2ee280ba3c9b6b1ec618c4ce6c9ca7 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 28 Jun 2024 11:16:50 +0200 Subject: [PATCH 10/17] Rename validate_data to validate_input_data --- google_sheets/app.py | 8 ++++---- google_sheets/data_processing/__init__.py | 4 ++-- google_sheets/data_processing/processing.py | 6 ++++-- tests/data_processing/test_processing.py | 6 +++--- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index 64696ae..64f4920 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -11,7 +11,7 @@ from googleapiclient.errors import HttpError from . import __version__ -from .data_processing import process_data_f, validate_data +from .data_processing import process_data_f, validate_input_data from .db_helpers import get_db_connection from .google_api import ( build_service, @@ -324,20 +324,20 @@ async def process_data( detail=f"Invalid data format. Please provide data in the correct format: {e}", ) from e - validation_error_msg = validate_data( + validation_error_msg = validate_input_data( df=new_campaign_df, mandatory_columns=NEW_CAMPAIGN_MANDATORY_COLUMNS, name="new campaign", ) if target_resource == "ad": - validation_error_msg += validate_data( + validation_error_msg += validate_input_data( df=template_df, mandatory_columns=MANDATORY_AD_TEMPLATE_COLUMNS, name="ads template", ) else: - validation_error_msg += validate_data( + validation_error_msg += validate_input_data( df=template_df, mandatory_columns=MANDATORY_KEYWORD_TEMPLATE_COLUMNS, name="keyword template", diff --git a/google_sheets/data_processing/__init__.py b/google_sheets/data_processing/__init__.py index 3718adf..4bb0c83 100644 --- a/google_sheets/data_processing/__init__.py +++ b/google_sheets/data_processing/__init__.py @@ -1,3 +1,3 @@ -from .processing import process_data_f, validate_data +from .processing import process_data_f, validate_input_data -__all__ = ["process_data_f", "validate_data"] +__all__ = ["process_data_f", "validate_input_data"] diff --git a/google_sheets/data_processing/processing.py b/google_sheets/data_processing/processing.py index e2fc895..aad11ec 100644 --- a/google_sheets/data_processing/processing.py +++ b/google_sheets/data_processing/processing.py @@ -4,10 +4,12 @@ from ..model import GoogleSheetValues -__all__ = ["process_data_f", "validate_data"] +__all__ = ["process_data_f", "validate_input_data"] -def validate_data(df: pd.DataFrame, mandatory_columns: List[str], name: str) -> str: +def validate_input_data( + df: pd.DataFrame, mandatory_columns: List[str], name: str +) -> str: error_msg = "" if len(df.columns) != len(set(df.columns)): error_msg = f"""Duplicate columns found in the {name} data. diff --git a/tests/data_processing/test_processing.py b/tests/data_processing/test_processing.py index b608e35..ae63cfb 100644 --- a/tests/data_processing/test_processing.py +++ b/tests/data_processing/test_processing.py @@ -3,7 +3,7 @@ import pandas as pd import pytest -from google_sheets.data_processing.processing import process_data_f, validate_data +from google_sheets.data_processing.processing import process_data_f, validate_input_data @pytest.mark.parametrize( @@ -41,9 +41,9 @@ ), ], ) -def test_validate_data(df: pd.DataFrame, expected: str) -> None: +def test_validate_input_data(df: pd.DataFrame, expected: str) -> None: mandatory_columns = ["Country", "Station From", "Station To"] - assert validate_data(df, mandatory_columns, "name") == expected + assert validate_input_data(df, mandatory_columns, "name") == expected @pytest.mark.parametrize( From 35d363946bc47ce0ebb0d4fb5406563f08b1380e Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 28 Jun 2024 11:27:58 +0200 Subject: [PATCH 11/17] Update process_data_f function to return DataFrame --- google_sheets/app.py | 6 +- google_sheets/data_processing/processing.py | 7 +- tests/data_processing/test_processing.py | 76 ++++++++++++++------- 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index 64f4920..e82810a 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -347,7 +347,11 @@ async def process_data( status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_msg ) - return process_data_f(template_df, new_campaign_df) + processed_df = process_data_f(template_df, new_campaign_df) + values = [processed_df.columns.tolist(), *processed_df.values.tolist()] + # validate_output_data(processed_values, target_resource) + + return GoogleSheetValues(values=values) @app.post( diff --git a/google_sheets/data_processing/processing.py b/google_sheets/data_processing/processing.py index aad11ec..60f03db 100644 --- a/google_sheets/data_processing/processing.py +++ b/google_sheets/data_processing/processing.py @@ -2,8 +2,6 @@ import pandas as pd -from ..model import GoogleSheetValues - __all__ = ["process_data_f", "validate_input_data"] @@ -30,7 +28,7 @@ def validate_input_data( def process_data_f( template_df: pd.DataFrame, new_campaign_df: pd.DataFrame -) -> GoogleSheetValues: +) -> pd.DataFrame: final_df = pd.DataFrame(columns=template_df.columns) for _, template_row in template_df.iterrows(): for _, new_campaign_row in new_campaign_df.iterrows(): @@ -63,5 +61,4 @@ def process_data_f( [final_df, pd.DataFrame([new_row])], ignore_index=True ) - values = [final_df.columns.tolist(), *final_df.values.tolist()] - return GoogleSheetValues(values=values) + return final_df diff --git a/tests/data_processing/test_processing.py b/tests/data_processing/test_processing.py index ae63cfb..8dbc2ba 100644 --- a/tests/data_processing/test_processing.py +++ b/tests/data_processing/test_processing.py @@ -65,17 +65,32 @@ def test_validate_input_data(df: pd.DataFrame, expected: str) -> None: "Station To": ["C", "D"], } ), - [ - ["Campaign", "Ad Group", "Keyword", "Max CPC"], - ["USA - A - C", "A - C", "k1", ""], - ["USA - A - C", "C - A", "k1", ""], - ["USA - B - D", "B - D", "k1", ""], - ["USA - B - D", "D - B", "k1", ""], - ["USA - A - C", "A - C", "k2", ""], - ["USA - A - C", "C - A", "k2", ""], - ["USA - B - D", "B - D", "k2", ""], - ["USA - B - D", "D - B", "k2", ""], - ], + pd.DataFrame( + { + "Campaign": [ + "USA - A - C", + "USA - A - C", + "USA - B - D", + "USA - B - D", + "USA - A - C", + "USA - A - C", + "USA - B - D", + "USA - B - D", + ], + "Ad Group": [ + "A - C", + "C - A", + "B - D", + "D - B", + "A - C", + "C - A", + "B - D", + "D - B", + ], + "Keyword": ["k1", "k1", "k1", "k1", "k2", "k2", "k2", "k2"], + "Max CPC": ["", "", "", "", "", "", "", ""], + } + ), ), ( pd.DataFrame( @@ -93,21 +108,36 @@ def test_validate_input_data(df: pd.DataFrame, expected: str) -> None: "Station To": ["C", "D"], } ), - [ - ["Campaign", "Ad Group", "Keyword", "Max CPC"], - ["USA - A - C", "A - C", "k1 A", ""], - ["USA - A - C", "C - A", "k1 C", ""], - ["USA - B - D", "B - D", "k1 B", ""], - ["USA - B - D", "D - B", "k1 D", ""], - ["USA - A - C", "A - C", "k2", ""], - ["USA - A - C", "C - A", "k2", ""], - ["USA - B - D", "B - D", "k2", ""], - ["USA - B - D", "D - B", "k2", ""], - ], + pd.DataFrame( + { + "Campaign": [ + "USA - A - C", + "USA - A - C", + "USA - B - D", + "USA - B - D", + "USA - A - C", + "USA - A - C", + "USA - B - D", + "USA - B - D", + ], + "Ad Group": [ + "A - C", + "C - A", + "B - D", + "D - B", + "A - C", + "C - A", + "B - D", + "D - B", + ], + "Keyword": ["k1 A", "k1 C", "k1 B", "k1 D", "k2", "k2", "k2", "k2"], + "Max CPC": ["", "", "", "", "", "", "", ""], + } + ), ), ], ) def test_process_data_f( template_df: pd.DataFrame, new_campaign_df: pd.DataFrame, expected: List[List[str]] ) -> None: - assert process_data_f(template_df, new_campaign_df).values == expected + process_data_f(template_df, new_campaign_df).equals(expected) From 3d59f340f9b1b426dde2c677c7fdcac2d00a339b Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 28 Jun 2024 13:05:46 +0200 Subject: [PATCH 12/17] wip _validate_output_data_ad --- google_sheets/data_processing/__init__.py | 4 +- google_sheets/data_processing/processing.py | 67 ++++++++++++++++++++- tests/data_processing/test_processing.py | 38 +++++++++++- 3 files changed, 104 insertions(+), 5 deletions(-) diff --git a/google_sheets/data_processing/__init__.py b/google_sheets/data_processing/__init__.py index 4bb0c83..03f8bd5 100644 --- a/google_sheets/data_processing/__init__.py +++ b/google_sheets/data_processing/__init__.py @@ -1,3 +1,3 @@ -from .processing import process_data_f, validate_input_data +from .processing import process_data_f, validate_input_data, validate_output_data -__all__ = ["process_data_f", "validate_input_data"] +__all__ = ["process_data_f", "validate_input_data", "validate_output_data"] diff --git a/google_sheets/data_processing/processing.py b/google_sheets/data_processing/processing.py index 60f03db..a992b0b 100644 --- a/google_sheets/data_processing/processing.py +++ b/google_sheets/data_processing/processing.py @@ -1,8 +1,8 @@ -from typing import List +from typing import List, Literal import pandas as pd -__all__ = ["process_data_f", "validate_input_data"] +__all__ = ["process_data_f", "validate_input_data", "validate_output_data"] def validate_input_data( @@ -62,3 +62,66 @@ def process_data_f( ) return final_df + + +MIN_HEADLINES = 3 +MAX_HEADLINES = 15 +MIN_DESCRIPTIONS = 2 +MAX_DESCRIPTIONS = 4 + + +def _validate_output_data_ad(df: pd.DataFrame) -> pd.DataFrame: + df["Issues"] = "" + headline_columns = [col for col in df.columns if "Headline" in col] + description_columns = [col for col in df.columns if "Description" in col] + + for index, row in df.iterrows(): + # Check for duplicate headlines and descriptions + if len(set(row[headline_columns])) != len(row[headline_columns]): + df.loc[index, "Issues"] += "Duplicate headlines found.\n" + if len(set(row[description_columns])) != len(row[description_columns]): + df.loc[index, "Issues"] += "Duplicate descriptions found.\n" + + # Check for the number of headlines and descriptions + headline_count = len( + [headline for headline in row[headline_columns] if headline] + ) + if headline_count < MIN_HEADLINES: + df.loc[index, "Issues"] += ( + f"Minimum {MIN_HEADLINES} headlines are required, found {headline_count}.\n" + ) + elif headline_count > MAX_HEADLINES: + df.loc[index, "Issues"] += ( + f"Maximum {MAX_HEADLINES} headlines are allowed, found {headline_count}.\n" + ) + + description_count = len( + [description for description in row[description_columns] if description] + ) + if description_count < MIN_DESCRIPTIONS: + df.loc[index, "Issues"] += ( + f"Minimum {MIN_DESCRIPTIONS} descriptions are required, found {description_count}.\n" + ) + elif description_count > MAX_DESCRIPTIONS: + df.loc[index, "Issues"] += ( + f"Maximum {MAX_DESCRIPTIONS} descriptions are allowed, found {description_count}.\n" + ) + + # Check for the final URL + if not row["Final URL"]: + df.loc[index, "Issues"] += "Final URL is missing.\n" + + if not df["Issues"].any(): + df = df.drop(columns=["Issues"]) + + return df + + +def validate_output_data( + df: pd.DataFrame, target_resource: Literal["ad", "keyword"] +) -> pd.DataFrame: + if target_resource == "keyword": + # No validation required for keyword data currently + return + + return _validate_output_data_ad(df) diff --git a/tests/data_processing/test_processing.py b/tests/data_processing/test_processing.py index 8dbc2ba..fd2b552 100644 --- a/tests/data_processing/test_processing.py +++ b/tests/data_processing/test_processing.py @@ -3,7 +3,11 @@ import pandas as pd import pytest -from google_sheets.data_processing.processing import process_data_f, validate_input_data +from google_sheets.data_processing.processing import ( + process_data_f, + validate_input_data, + validate_output_data, +) @pytest.mark.parametrize( @@ -141,3 +145,35 @@ def test_process_data_f( template_df: pd.DataFrame, new_campaign_df: pd.DataFrame, expected: List[List[str]] ) -> None: process_data_f(template_df, new_campaign_df).equals(expected) + + +def test_validate_output_data() -> None: + df = pd.DataFrame( + { + "Headline 1": ["H1", "H1", "H1", "H1"], + "Headline 2": ["H1", "H2", "H2", "H2"], + "Headline 3": ["H3", "H3", "H3", ""], + "Description 1": ["D1", "D1", "D2", "D3"], + "Description 2": ["D1", "D1", "D3", ""], + "Final URL": ["F1", "F1", "F1", ""], + } + ) + result = validate_output_data(df, "ad") + expected = pd.DataFrame( + { + "Headline 1": ["H1", "H1", "H1", "H1"], + "Headline 2": ["H1", "H2", "H2", "H2"], + "Headline 3": ["H3", "H3", "H3", ""], + "Description 1": ["D1", "D1", "D2", "D3"], + "Description 2": ["D1", "D1", "D3", ""], + "Final URL": ["F1", "F1", "F1", ""], + "Issues": [ + "Duplicate headlines found.\nDuplicate descriptions found.\n", + "Duplicate descriptions found.\n", + "", + "Minimum 3 headlines are required, found 2.\nMinimum 2 descriptions are required, found 1.\nFinal URL is missing.\n", + ], + } + ) + + assert result.equals(expected) From ad724e132b5fd0fc02d1f11984bb91eba12454ca Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 28 Jun 2024 13:32:45 +0200 Subject: [PATCH 13/17] wip --- google_sheets/app.py | 8 +++-- google_sheets/data_processing/processing.py | 8 ++--- tests/data_processing/test_processing.py | 36 ++++++++++----------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index e82810a..a0dfa25 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -11,7 +11,7 @@ from googleapiclient.errors import HttpError from . import __version__ -from .data_processing import process_data_f, validate_input_data +from .data_processing import process_data_f, validate_input_data, validate_output_data from .db_helpers import get_db_connection from .google_api import ( build_service, @@ -348,8 +348,10 @@ async def process_data( ) processed_df = process_data_f(template_df, new_campaign_df) - values = [processed_df.columns.tolist(), *processed_df.values.tolist()] - # validate_output_data(processed_values, target_resource) + + validated_df = validate_output_data(processed_df, target_resource) + + values = [validated_df.columns.tolist(), *validated_df.values.tolist()] return GoogleSheetValues(values=values) diff --git a/google_sheets/data_processing/processing.py b/google_sheets/data_processing/processing.py index a992b0b..d70d575 100644 --- a/google_sheets/data_processing/processing.py +++ b/google_sheets/data_processing/processing.py @@ -107,9 +107,9 @@ def _validate_output_data_ad(df: pd.DataFrame) -> pd.DataFrame: f"Maximum {MAX_DESCRIPTIONS} descriptions are allowed, found {description_count}.\n" ) - # Check for the final URL - if not row["Final URL"]: - df.loc[index, "Issues"] += "Final URL is missing.\n" + # TODO: Check for the final URL + # if not row["Final URL"]: + # df.loc[index, "Issues"] += "Final URL is missing.\n" if not df["Issues"].any(): df = df.drop(columns=["Issues"]) @@ -122,6 +122,6 @@ def validate_output_data( ) -> pd.DataFrame: if target_resource == "keyword": # No validation required for keyword data currently - return + return df return _validate_output_data_ad(df) diff --git a/tests/data_processing/test_processing.py b/tests/data_processing/test_processing.py index fd2b552..66e2065 100644 --- a/tests/data_processing/test_processing.py +++ b/tests/data_processing/test_processing.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional import pandas as pd import pytest @@ -147,7 +147,19 @@ def test_process_data_f( process_data_f(template_df, new_campaign_df).equals(expected) -def test_validate_output_data() -> None: +@pytest.mark.parametrize( + "issues_column", + [ + [ + "Duplicate headlines found.\nDuplicate descriptions found.\n", + "Duplicate descriptions found.\n", + "", + "Minimum 3 headlines are required, found 2.\nMinimum 2 descriptions are required, found 1.\n", + ], + None, + ], +) +def test_validate_output_data(issues_column: Optional[List[str]]) -> None: df = pd.DataFrame( { "Headline 1": ["H1", "H1", "H1", "H1"], @@ -155,25 +167,11 @@ def test_validate_output_data() -> None: "Headline 3": ["H3", "H3", "H3", ""], "Description 1": ["D1", "D1", "D2", "D3"], "Description 2": ["D1", "D1", "D3", ""], - "Final URL": ["F1", "F1", "F1", ""], } ) result = validate_output_data(df, "ad") - expected = pd.DataFrame( - { - "Headline 1": ["H1", "H1", "H1", "H1"], - "Headline 2": ["H1", "H2", "H2", "H2"], - "Headline 3": ["H3", "H3", "H3", ""], - "Description 1": ["D1", "D1", "D2", "D3"], - "Description 2": ["D1", "D1", "D3", ""], - "Final URL": ["F1", "F1", "F1", ""], - "Issues": [ - "Duplicate headlines found.\nDuplicate descriptions found.\n", - "Duplicate descriptions found.\n", - "", - "Minimum 3 headlines are required, found 2.\nMinimum 2 descriptions are required, found 1.\nFinal URL is missing.\n", - ], - } - ) + expected = df.copy() + if issues_column: + expected["Issues"] = issues_column assert result.equals(expected) From 688594f4b23dbd66b3c65004e9f2baf791b49245 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 28 Jun 2024 15:44:24 +0200 Subject: [PATCH 14/17] Update ads dataframe validation --- google_sheets/data_processing/processing.py | 20 +++++++++++++++++++- tests/data_processing/test_processing.py | 4 ++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/google_sheets/data_processing/processing.py b/google_sheets/data_processing/processing.py index d70d575..fe50755 100644 --- a/google_sheets/data_processing/processing.py +++ b/google_sheets/data_processing/processing.py @@ -69,8 +69,11 @@ def process_data_f( MIN_DESCRIPTIONS = 2 MAX_DESCRIPTIONS = 4 +MAX_HEADLINE_LENGTH = 30 +MAX_DESCRIPTION_LENGTH = 90 -def _validate_output_data_ad(df: pd.DataFrame) -> pd.DataFrame: + +def _validate_output_data_ad(df: pd.DataFrame) -> pd.DataFrame: # noqa: C901 df["Issues"] = "" headline_columns = [col for col in df.columns if "Headline" in col] description_columns = [col for col in df.columns if "Description" in col] @@ -107,6 +110,21 @@ def _validate_output_data_ad(df: pd.DataFrame) -> pd.DataFrame: f"Maximum {MAX_DESCRIPTIONS} descriptions are allowed, found {description_count}.\n" ) + # Check for the length of headlines and descriptions + for headline_column in headline_columns: + headline = row[headline_column] + if len(headline) > MAX_HEADLINE_LENGTH: + df.loc[index, "Issues"] += ( + f"Headline length should be less than {MAX_HEADLINE_LENGTH} characters, found {len(headline)} in column {headline_column}.\n" + ) + + for description_column in description_columns: + description = row[description_column] + if len(description) > MAX_DESCRIPTION_LENGTH: + df.loc[index, "Issues"] += ( + f"Description length should be less than {MAX_DESCRIPTION_LENGTH} characters, found {len(description)} in column {description_column}.\n" + ) + # TODO: Check for the final URL # if not row["Final URL"]: # df.loc[index, "Issues"] += "Final URL is missing.\n" diff --git a/tests/data_processing/test_processing.py b/tests/data_processing/test_processing.py index 66e2065..089a397 100644 --- a/tests/data_processing/test_processing.py +++ b/tests/data_processing/test_processing.py @@ -154,7 +154,7 @@ def test_process_data_f( "Duplicate headlines found.\nDuplicate descriptions found.\n", "Duplicate descriptions found.\n", "", - "Minimum 3 headlines are required, found 2.\nMinimum 2 descriptions are required, found 1.\n", + "Minimum 3 headlines are required, found 2.\nMinimum 2 descriptions are required, found 1.\nHeadline length should be less than 30 characters, found 31 in column Headline 2.\n", ], None, ], @@ -163,7 +163,7 @@ def test_validate_output_data(issues_column: Optional[List[str]]) -> None: df = pd.DataFrame( { "Headline 1": ["H1", "H1", "H1", "H1"], - "Headline 2": ["H1", "H2", "H2", "H2"], + "Headline 2": ["H1", "H2", "H2", ("H" * 31)], "Headline 3": ["H3", "H3", "H3", ""], "Description 1": ["D1", "D1", "D2", "D3"], "Description 2": ["D1", "D1", "D3", ""], From d140419da85f2489631805d3a88b3defd9b16d3d Mon Sep 17 00:00:00 2001 From: Davor Runje Date: Mon, 1 Jul 2024 07:50:40 +0000 Subject: [PATCH 15/17] .devcontainer added --- .devcontainer/devcontainer.env | 4 ++ .devcontainer/python-3.10/devcontainer.json | 57 ++++++++++++++++++++ .devcontainer/python-3.10/docker-compose.yml | 30 +++++++++++ .devcontainer/python-3.11/devcontainer.json | 57 ++++++++++++++++++++ .devcontainer/python-3.11/docker-compose.yml | 30 +++++++++++ .devcontainer/python-3.12/devcontainer.json | 57 ++++++++++++++++++++ .devcontainer/python-3.12/docker-compose.yml | 30 +++++++++++ .devcontainer/python-3.9/devcontainer.json | 57 ++++++++++++++++++++ .devcontainer/python-3.9/docker-compose.yml | 30 +++++++++++ .devcontainer/setup.sh | 26 +++++++++ 10 files changed, 378 insertions(+) create mode 100644 .devcontainer/devcontainer.env create mode 100644 .devcontainer/python-3.10/devcontainer.json create mode 100644 .devcontainer/python-3.10/docker-compose.yml create mode 100644 .devcontainer/python-3.11/devcontainer.json create mode 100644 .devcontainer/python-3.11/docker-compose.yml create mode 100644 .devcontainer/python-3.12/devcontainer.json create mode 100644 .devcontainer/python-3.12/docker-compose.yml create mode 100644 .devcontainer/python-3.9/devcontainer.json create mode 100644 .devcontainer/python-3.9/docker-compose.yml create mode 100644 .devcontainer/setup.sh diff --git a/.devcontainer/devcontainer.env b/.devcontainer/devcontainer.env new file mode 100644 index 0000000..31bf13c --- /dev/null +++ b/.devcontainer/devcontainer.env @@ -0,0 +1,4 @@ +PORT_PREFIX=${PORT_PREFIX} +CONTAINER_PREFIX=${USER} + +GOOGLE_SHEETS_CLIENT_SECRET=${GOOGLE_SHEETS_CLIENT_SECRET} diff --git a/.devcontainer/python-3.10/devcontainer.json b/.devcontainer/python-3.10/devcontainer.json new file mode 100644 index 0000000..696ea7d --- /dev/null +++ b/.devcontainer/python-3.10/devcontainer.json @@ -0,0 +1,57 @@ +{ + "name": "python-3.10", + "dockerComposeFile": [ + "./docker-compose.yml" + ], + "service": "python-3.10", + "forwardPorts": [], + "shutdownAction": "stopCompose", + "workspaceFolder": "/workspaces/google-sheets", + "remoteEnv": {}, + "features": { + "ghcr.io/devcontainers/features/common-utils:2": { + "installZsh": true, + "installOhMyZsh": true, + "configureZshAsDefaultShell": true, + "username": "vscode", + "userUid": "1000", + "userGid": "1000" + }, + "ghcr.io/devcontainers/features/node:1": {}, + "ghcr.io/devcontainers/features/git:1": { + "version": "latest", + "ppa": true + }, + "ghcr.io/devcontainers/features/git-lfs:1": {}, + "ghcr.io/robbert229/devcontainer-features/postgresql-client:1": {} + }, + "updateContentCommand": "bash .devcontainer/setup.sh", + "postCreateCommand": [], + "customizations": { + "vscode": { + "settings": { + "python.linting.enabled": true, + "python.testing.pytestEnabled": true, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": "always" + }, + "[python]": { + "editor.defaultFormatter": "ms-python.vscode-pylance" + }, + "editor.rulers": [ + 80 + ] + }, + "extensions": [ + "ms-python.python", + "ms-toolsai.jupyter", + "ms-toolsai.vscode-jupyter-cell-tags", + "ms-toolsai.jupyter-keymap", + "ms-toolsai.jupyter-renderers", + "ms-toolsai.vscode-jupyter-slideshow", + "ms-python.vscode-pylance" + ] + } + } +} diff --git a/.devcontainer/python-3.10/docker-compose.yml b/.devcontainer/python-3.10/docker-compose.yml new file mode 100644 index 0000000..ce1a780 --- /dev/null +++ b/.devcontainer/python-3.10/docker-compose.yml @@ -0,0 +1,30 @@ +version: '3' + +services: + python-3.10: # nosemgrep + image: mcr.microsoft.com/devcontainers/python:3.10 + container_name: $USER-python-3.10-google-sheets + volumes: + - ../../:/workspaces/google-sheets:cached + command: sleep infinity + environment: + - DATABASE_URL=postgresql://admin:password@${USER}-postgres-py39-google-sheets:5432/google-sheets + env_file: + - ../devcontainer.env + networks: + - google-sheets-network + postgres-google-sheets: # nosemgrep + image: postgres:latest + container_name: $USER-postgres-py39-google-sheets + environment: + POSTGRES_USER: admin + POSTGRES_PASSWORD: password # pragma: allowlist secret + POSTGRES_DB: google-sheets + # ports: + # - "${PORT_PREFIX}5432:5432" + networks: + - google-sheets-network + +networks: + google-sheets-network: + name: "${USER}-google-sheets-network" diff --git a/.devcontainer/python-3.11/devcontainer.json b/.devcontainer/python-3.11/devcontainer.json new file mode 100644 index 0000000..4511947 --- /dev/null +++ b/.devcontainer/python-3.11/devcontainer.json @@ -0,0 +1,57 @@ +{ + "name": "python-3.11", + "dockerComposeFile": [ + "./docker-compose.yml" + ], + "service": "python-3.11", + "forwardPorts": [], + "shutdownAction": "stopCompose", + "workspaceFolder": "/workspaces/google-sheets", + "remoteEnv": {}, + "features": { + "ghcr.io/devcontainers/features/common-utils:2": { + "installZsh": true, + "installOhMyZsh": true, + "configureZshAsDefaultShell": true, + "username": "vscode", + "userUid": "1000", + "userGid": "1000" + }, + "ghcr.io/devcontainers/features/node:1": {}, + "ghcr.io/devcontainers/features/git:1": { + "version": "latest", + "ppa": true + }, + "ghcr.io/devcontainers/features/git-lfs:1": {}, + "ghcr.io/robbert229/devcontainer-features/postgresql-client:1": {} + }, + "updateContentCommand": "bash .devcontainer/setup.sh", + "postCreateCommand": [], + "customizations": { + "vscode": { + "settings": { + "python.linting.enabled": true, + "python.testing.pytestEnabled": true, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": "always" + }, + "[python]": { + "editor.defaultFormatter": "ms-python.vscode-pylance" + }, + "editor.rulers": [ + 80 + ] + }, + "extensions": [ + "ms-python.python", + "ms-toolsai.jupyter", + "ms-toolsai.vscode-jupyter-cell-tags", + "ms-toolsai.jupyter-keymap", + "ms-toolsai.jupyter-renderers", + "ms-toolsai.vscode-jupyter-slideshow", + "ms-python.vscode-pylance" + ] + } + } +} diff --git a/.devcontainer/python-3.11/docker-compose.yml b/.devcontainer/python-3.11/docker-compose.yml new file mode 100644 index 0000000..35de7bc --- /dev/null +++ b/.devcontainer/python-3.11/docker-compose.yml @@ -0,0 +1,30 @@ +version: '3' + +services: + python-3.11: # nosemgrep + image: mcr.microsoft.com/devcontainers/python:3.11 + container_name: $USER-python-3.11-google-sheets + volumes: + - ../../:/workspaces/google-sheets:cached + command: sleep infinity + environment: + - DATABASE_URL=postgresql://admin:password@${USER}-postgres-py39-google-sheets:5432/google-sheets + env_file: + - ../devcontainer.env + networks: + - google-sheets-network + postgres-google-sheets: # nosemgrep + image: postgres:latest + container_name: $USER-postgres-py39-google-sheets + environment: + POSTGRES_USER: admin + POSTGRES_PASSWORD: password # pragma: allowlist secret + POSTGRES_DB: google-sheets + # ports: + # - "${PORT_PREFIX}5432:5432" + networks: + - google-sheets-network + +networks: + google-sheets-network: + name: "${USER}-google-sheets-network" diff --git a/.devcontainer/python-3.12/devcontainer.json b/.devcontainer/python-3.12/devcontainer.json new file mode 100644 index 0000000..4f177cd --- /dev/null +++ b/.devcontainer/python-3.12/devcontainer.json @@ -0,0 +1,57 @@ +{ + "name": "python-3.12", + "dockerComposeFile": [ + "./docker-compose.yml" + ], + "service": "python-3.12", + "forwardPorts": [], + "shutdownAction": "stopCompose", + "workspaceFolder": "/workspaces/google-sheets", + "remoteEnv": {}, + "features": { + "ghcr.io/devcontainers/features/common-utils:2": { + "installZsh": true, + "installOhMyZsh": true, + "configureZshAsDefaultShell": true, + "username": "vscode", + "userUid": "1000", + "userGid": "1000" + }, + "ghcr.io/devcontainers/features/node:1": {}, + "ghcr.io/devcontainers/features/git:1": { + "version": "latest", + "ppa": true + }, + "ghcr.io/devcontainers/features/git-lfs:1": {}, + "ghcr.io/robbert229/devcontainer-features/postgresql-client:1": {} + }, + "updateContentCommand": "bash .devcontainer/setup.sh", + "postCreateCommand": [], + "customizations": { + "vscode": { + "settings": { + "python.linting.enabled": true, + "python.testing.pytestEnabled": true, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": "always" + }, + "[python]": { + "editor.defaultFormatter": "ms-python.vscode-pylance" + }, + "editor.rulers": [ + 80 + ] + }, + "extensions": [ + "ms-python.python", + "ms-toolsai.jupyter", + "ms-toolsai.vscode-jupyter-cell-tags", + "ms-toolsai.jupyter-keymap", + "ms-toolsai.jupyter-renderers", + "ms-toolsai.vscode-jupyter-slideshow", + "ms-python.vscode-pylance" + ] + } + } +} diff --git a/.devcontainer/python-3.12/docker-compose.yml b/.devcontainer/python-3.12/docker-compose.yml new file mode 100644 index 0000000..82b25d6 --- /dev/null +++ b/.devcontainer/python-3.12/docker-compose.yml @@ -0,0 +1,30 @@ +version: '3' + +services: + python-3.12: # nosemgrep + image: mcr.microsoft.com/devcontainers/python:3.12 + container_name: $USER-python-3.12-google-sheets + volumes: + - ../../:/workspaces/google-sheets:cached + command: sleep infinity + environment: + - DATABASE_URL=postgresql://admin:password@${USER}-postgres-py39-google-sheets:5432/google-sheets + env_file: + - ../devcontainer.env + networks: + - google-sheets-network + postgres-google-sheets: # nosemgrep + image: postgres:latest + container_name: $USER-postgres-py39-google-sheets + environment: + POSTGRES_USER: admin + POSTGRES_PASSWORD: password # pragma: allowlist secret + POSTGRES_DB: google-sheets + # ports: + # - "${PORT_PREFIX}5432:5432" + networks: + - google-sheets-network + +networks: + google-sheets-network: + name: "${USER}-google-sheets-network" diff --git a/.devcontainer/python-3.9/devcontainer.json b/.devcontainer/python-3.9/devcontainer.json new file mode 100644 index 0000000..fe45a9e --- /dev/null +++ b/.devcontainer/python-3.9/devcontainer.json @@ -0,0 +1,57 @@ +{ + "name": "python-3.9", + "dockerComposeFile": [ + "./docker-compose.yml" + ], + "service": "python-3.9", + "forwardPorts": [], + "shutdownAction": "stopCompose", + "workspaceFolder": "/workspaces/google-sheets", + "remoteEnv": {}, + "features": { + "ghcr.io/devcontainers/features/common-utils:2": { + "installZsh": true, + "installOhMyZsh": true, + "configureZshAsDefaultShell": true, + "username": "vscode", + "userUid": "1000", + "userGid": "1000" + }, + "ghcr.io/devcontainers/features/node:1": {}, + "ghcr.io/devcontainers/features/git:1": { + "version": "latest", + "ppa": true + }, + "ghcr.io/devcontainers/features/git-lfs:1": {}, + "ghcr.io/robbert229/devcontainer-features/postgresql-client:1": {} + }, + "updateContentCommand": "bash .devcontainer/setup.sh", + "postCreateCommand": [], + "customizations": { + "vscode": { + "settings": { + "python.linting.enabled": true, + "python.testing.pytestEnabled": true, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": "always" + }, + "[python]": { + "editor.defaultFormatter": "ms-python.vscode-pylance" + }, + "editor.rulers": [ + 80 + ] + }, + "extensions": [ + "ms-python.python", + "ms-toolsai.jupyter", + "ms-toolsai.vscode-jupyter-cell-tags", + "ms-toolsai.jupyter-keymap", + "ms-toolsai.jupyter-renderers", + "ms-toolsai.vscode-jupyter-slideshow", + "ms-python.vscode-pylance" + ] + } + } +} diff --git a/.devcontainer/python-3.9/docker-compose.yml b/.devcontainer/python-3.9/docker-compose.yml new file mode 100644 index 0000000..342c8e7 --- /dev/null +++ b/.devcontainer/python-3.9/docker-compose.yml @@ -0,0 +1,30 @@ +version: '3' + +services: + python-3.9: # nosemgrep + image: mcr.microsoft.com/devcontainers/python:3.9 + container_name: $USER-python-3.9-google-sheets + volumes: + - ../../:/workspaces/google-sheets:cached + command: sleep infinity + environment: + - DATABASE_URL=postgresql://admin:password@${USER}-postgres-py39-google-sheets:5432/google-sheets + env_file: + - ../devcontainer.env + networks: + - google-sheets-network + postgres-google-sheets: # nosemgrep + image: postgres:latest + container_name: $USER-postgres-py39-google-sheets + environment: + POSTGRES_USER: admin + POSTGRES_PASSWORD: password # pragma: allowlist secret + POSTGRES_DB: google-sheets + # ports: + # - "${PORT_PREFIX}5432:5432" + networks: + - google-sheets-network + +networks: + google-sheets-network: + name: "${USER}-google-sheets-network" diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100644 index 0000000..3a9afaa --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,26 @@ +# update pip +pip install --upgrade pip + +# install dev packages +pip install -e ".[dev]" + +# install pre-commit hook if not installed already +pre-commit install + +prisma migrate deploy +prisma generate + +echo '{ + "web": { + "client_id": "1027914582771-g0bcsn4fhd6a59pp3d4n1ntjc03r1k9s.apps.googleusercontent.com", + "project_id": "captn-sheets-dev", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_secret": "'${GOOGLE_SHEETS_CLIENT_SECRET}'", + "redirect_uris": [ + "http://localhost:8000/login/callback" + ] + + } +}' > client_secret.json From 300615dc4ee77ebaff1ba3135e6430d70cddab8b Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Mon, 1 Jul 2024 11:58:41 +0000 Subject: [PATCH 16/17] Update missing FastAPI endpoint/query descriptions --- google_sheets/app.py | 18 +- tests/app/fixtures/openapi.json | 695 +++++++++++++++++++++++++++++++- 2 files changed, 705 insertions(+), 8 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index a0dfa25..d695690 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -6,7 +6,7 @@ import httpx import pandas as pd -from fastapi import FastAPI, HTTPException, Query, Request, Response, status +from fastapi import FastAPI, HTTPException, Query, Response, status from fastapi.responses import RedirectResponse from googleapiclient.errors import HttpError @@ -54,11 +54,12 @@ async def is_authenticated_for_ads(user_id: int) -> bool: # Route 1: Redirect to Google OAuth -@app.get("/login") +@app.get("/login", description="Get the URL to log in with Google") async def get_login_url( - request: Request, - user_id: int = Query(title="User ID"), - force_new_login: bool = Query(title="Force new login", default=False), + user_id: Annotated[ + int, Query(description="The user ID for which the data is requested") + ], + force_new_login: Annotated[bool, Query(description="Force new login")] = False, ) -> Dict[str, str]: if not force_new_login: is_authenticated = await is_authenticated_for_ads(user_id=user_id) @@ -70,7 +71,7 @@ async def get_login_url( return {"login_url": markdown_url} -@app.get("/login/success") +@app.get("/login/success", description="Get the success message after login") async def get_login_success() -> Dict[str, str]: return {"login_success": "You have successfully logged in"} @@ -78,7 +79,10 @@ async def get_login_success() -> Dict[str, str]: # Route 2: Save user credentials/token to a JSON file @app.get("/login/callback") async def login_callback( - code: str = Query(title="Authorization Code"), state: str = Query(title="State") + code: Annotated[ + str, Query(description="The authorization code received after successful login") + ], + state: Annotated[str, Query(description="State")], ) -> RedirectResponse: if not state.isdigit(): raise HTTPException(status_code=400, detail="User ID must be an integer") diff --git a/tests/app/fixtures/openapi.json b/tests/app/fixtures/openapi.json index dae4ed4..af7fe1d 100644 --- a/tests/app/fixtures/openapi.json +++ b/tests/app/fixtures/openapi.json @@ -1 +1,694 @@ -{"openapi":"3.1.0","info":{"title":"google-sheets","version":"0.1.0"},"servers":[{"url":"http://localhost:8000","description":"Google Sheets app server"}],"paths":{"/login":{"get":{"summary":"Get Login Url","operationId":"get_login_url_login_get","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","title":"User ID"}},{"name":"force_new_login","in":"query","required":false,"schema":{"type":"boolean","title":"Force new login","default":false}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"type":"object","additionalProperties":{"type":"string"},"title":"Response Get Login Url Login Get"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/login/success":{"get":{"summary":"Get Login Success","operationId":"get_login_success_login_success_get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"additionalProperties":{"type":"string"},"type":"object","title":"Response Get Login Success Login Success Get"}}}}}}},"/login/callback":{"get":{"summary":"Login Callback","operationId":"login_callback_login_callback_get","parameters":[{"name":"code","in":"query","required":true,"schema":{"type":"string","title":"Authorization Code"}},{"name":"state","in":"query","required":true,"schema":{"type":"string","title":"State"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/get-sheet":{"get":{"summary":"Get Sheet","description":"Get data from a Google Sheet","operationId":"get_sheet_get_sheet_get","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","description":"The user ID for which the data is requested","title":"User Id"},"description":"The user ID for which the data is requested"},{"name":"spreadsheet_id","in":"query","required":true,"schema":{"type":"string","description":"ID of the Google Sheet to fetch data from","title":"Spreadsheet Id"},"description":"ID of the Google Sheet to fetch data from"},{"name":"title","in":"query","required":true,"schema":{"type":"string","description":"The title of the sheet to fetch data from","title":"Title"},"description":"The title of the sheet to fetch data from"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"anyOf":[{"type":"string"},{"$ref":"#/components/schemas/GoogleSheetValues"}],"title":"Response Get Sheet Get Sheet Get"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/update-sheet":{"post":{"summary":"Update Sheet","description":"Update data in a Google Sheet within the existing spreadsheet","operationId":"update_sheet_update_sheet_post","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","description":"The user ID for which the data is requested","title":"User Id"},"description":"The user ID for which the data is requested"},{"name":"spreadsheet_id","in":"query","required":true,"schema":{"type":"string","description":"ID of the Google Sheet to fetch data from","title":"Spreadsheet Id"},"description":"ID of the Google Sheet to fetch data from"},{"name":"title","in":"query","required":true,"schema":{"type":"string","description":"The title of the sheet to update","title":"Title"},"description":"The title of the sheet to update"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GoogleSheetValues"}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/create-sheet":{"post":{"summary":"Create Sheet","description":"Create a new Google Sheet within the existing spreadsheet","operationId":"create_sheet_create_sheet_post","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","description":"The user ID for which the data is requested","title":"User Id"},"description":"The user ID for which the data is requested"},{"name":"spreadsheet_id","in":"query","required":true,"schema":{"type":"string","description":"ID of the Google Sheet to fetch data from","title":"Spreadsheet Id"},"description":"ID of the Google Sheet to fetch data from"},{"name":"title","in":"query","required":true,"schema":{"type":"string","description":"The title of the new sheet","title":"Title"},"description":"The title of the new sheet"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/get-all-file-names":{"get":{"summary":"Get All File Names","description":"Get all sheets associated with the user","operationId":"get_all_file_names_get_all_file_names_get","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","description":"The user ID for which the data is requested","title":"User Id"},"description":"The user ID for which the data is requested"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"type":"object","additionalProperties":{"type":"string"},"title":"Response Get All File Names Get All File Names Get"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/get-all-sheet-titles":{"get":{"summary":"Get All Sheet Titles","description":"Get all sheet titles within a Google Spreadsheet","operationId":"get_all_sheet_titles_get_all_sheet_titles_get","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","description":"The user ID for which the data is requested","title":"User Id"},"description":"The user ID for which the data is requested"},{"name":"spreadsheet_id","in":"query","required":true,"schema":{"type":"string","description":"ID of the Google Sheet to fetch data from","title":"Spreadsheet Id"},"description":"ID of the Google Sheet to fetch data from"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"type":"array","items":{"type":"string"},"title":"Response Get All Sheet Titles Get All Sheet Titles Get"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/process-data":{"post":{"summary":"Process Data","description":"Process data to generate new ads or keywords based on the template","operationId":"process_data_process_data_post","parameters":[{"name":"target_resource","in":"query","required":true,"schema":{"enum":["ad","keyword"],"type":"string","description":"The target resource to be updated","title":"Target Resource"},"description":"The target resource to be updated"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Body_process_data_process_data_post"}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GoogleSheetValues"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/process-spreadsheet":{"post":{"summary":"Process Spreadsheet","description":"Process data to generate new ads or keywords based on the template","operationId":"process_spreadsheet_process_spreadsheet_post","parameters":[{"name":"user_id","in":"query","required":true,"schema":{"type":"integer","description":"The user ID for which the data is requested","title":"User Id"},"description":"The user ID for which the data is requested"},{"name":"template_spreadsheet_id","in":"query","required":true,"schema":{"type":"string","description":"ID of the Google Sheet with the template data","title":"Template Spreadsheet Id"},"description":"ID of the Google Sheet with the template data"},{"name":"template_sheet_title","in":"query","required":true,"schema":{"type":"string","description":"The title of the sheet with the template data","title":"Template Sheet Title"},"description":"The title of the sheet with the template data"},{"name":"new_campaign_spreadsheet_id","in":"query","required":true,"schema":{"type":"string","description":"ID of the Google Sheet with the new campaign data","title":"New Campaign Spreadsheet Id"},"description":"ID of the Google Sheet with the new campaign data"},{"name":"new_campaign_sheet_title","in":"query","required":true,"schema":{"type":"string","description":"The title of the sheet with the new campaign data","title":"New Campaign Sheet Title"},"description":"The title of the sheet with the new campaign data"},{"name":"target_resource","in":"query","required":true,"schema":{"enum":["ad","keyword"],"type":"string","description":"The target resource to be updated","title":"Target Resource"},"description":"The target resource to be updated"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}}},"components":{"schemas":{"Body_process_data_process_data_post":{"properties":{"template_sheet_values":{"$ref":"#/components/schemas/GoogleSheetValues"},"new_campaign_sheet_values":{"$ref":"#/components/schemas/GoogleSheetValues"}},"type":"object","required":["template_sheet_values","new_campaign_sheet_values"],"title":"Body_process_data_process_data_post"},"GoogleSheetValues":{"properties":{"values":{"items":{"items":{},"type":"array"},"type":"array","title":"Values","description":"Values to be written to the Google Sheet."}},"type":"object","required":["values"],"title":"GoogleSheetValues"},"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"}}}} +{ + "openapi": "3.1.0", + "info": { + "title": "google-sheets", + "version": "0.1.0" + }, + "servers": [ + { + "url": "http://localhost:8000", + "description": "Google Sheets app server" + } + ], + "paths": { + "/login": { + "get": { + "summary": "Get Login Url", + "description": "Get the URL to log in with Google", + "operationId": "get_login_url_login_get", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "description": "The user ID for which the data is requested", + "title": "User Id" + }, + "description": "The user ID for which the data is requested" + }, + { + "name": "force_new_login", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "description": "Force new login", + "default": false, + "title": "Force New Login" + }, + "description": "Force new login" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "title": "Response Get Login Url Login Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/login/success": { + "get": { + "summary": "Get Login Success", + "description": "Get the success message after login", + "operationId": "get_login_success_login_success_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "title": "Response Get Login Success Login Success Get" + } + } + } + } + } + } + }, + "/login/callback": { + "get": { + "summary": "Login Callback", + "operationId": "login_callback_login_callback_get", + "parameters": [ + { + "name": "code", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "The authorization code received after successful login", + "title": "Code" + }, + "description": "The authorization code received after successful login" + }, + { + "name": "state", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "State", + "title": "State" + }, + "description": "State" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/get-sheet": { + "get": { + "summary": "Get Sheet", + "description": "Get data from a Google Sheet", + "operationId": "get_sheet_get_sheet_get", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "description": "The user ID for which the data is requested", + "title": "User Id" + }, + "description": "The user ID for which the data is requested" + }, + { + "name": "spreadsheet_id", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "ID of the Google Sheet to fetch data from", + "title": "Spreadsheet Id" + }, + "description": "ID of the Google Sheet to fetch data from" + }, + { + "name": "title", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "The title of the sheet to fetch data from", + "title": "Title" + }, + "description": "The title of the sheet to fetch data from" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/components/schemas/GoogleSheetValues" + } + ], + "title": "Response Get Sheet Get Sheet Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/update-sheet": { + "post": { + "summary": "Update Sheet", + "description": "Update data in a Google Sheet within the existing spreadsheet", + "operationId": "update_sheet_update_sheet_post", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "description": "The user ID for which the data is requested", + "title": "User Id" + }, + "description": "The user ID for which the data is requested" + }, + { + "name": "spreadsheet_id", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "ID of the Google Sheet to fetch data from", + "title": "Spreadsheet Id" + }, + "description": "ID of the Google Sheet to fetch data from" + }, + { + "name": "title", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "The title of the sheet to update", + "title": "Title" + }, + "description": "The title of the sheet to update" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GoogleSheetValues" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/create-sheet": { + "post": { + "summary": "Create Sheet", + "description": "Create a new Google Sheet within the existing spreadsheet", + "operationId": "create_sheet_create_sheet_post", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "description": "The user ID for which the data is requested", + "title": "User Id" + }, + "description": "The user ID for which the data is requested" + }, + { + "name": "spreadsheet_id", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "ID of the Google Sheet to fetch data from", + "title": "Spreadsheet Id" + }, + "description": "ID of the Google Sheet to fetch data from" + }, + { + "name": "title", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "The title of the new sheet", + "title": "Title" + }, + "description": "The title of the new sheet" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/get-all-file-names": { + "get": { + "summary": "Get All File Names", + "description": "Get all sheets associated with the user", + "operationId": "get_all_file_names_get_all_file_names_get", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "description": "The user ID for which the data is requested", + "title": "User Id" + }, + "description": "The user ID for which the data is requested" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "title": "Response Get All File Names Get All File Names Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/get-all-sheet-titles": { + "get": { + "summary": "Get All Sheet Titles", + "description": "Get all sheet titles within a Google Spreadsheet", + "operationId": "get_all_sheet_titles_get_all_sheet_titles_get", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "description": "The user ID for which the data is requested", + "title": "User Id" + }, + "description": "The user ID for which the data is requested" + }, + { + "name": "spreadsheet_id", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "ID of the Google Sheet to fetch data from", + "title": "Spreadsheet Id" + }, + "description": "ID of the Google Sheet to fetch data from" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "title": "Response Get All Sheet Titles Get All Sheet Titles Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/process-data": { + "post": { + "summary": "Process Data", + "description": "Process data to generate new ads or keywords based on the template", + "operationId": "process_data_process_data_post", + "parameters": [ + { + "name": "target_resource", + "in": "query", + "required": true, + "schema": { + "enum": [ + "ad", + "keyword" + ], + "type": "string", + "description": "The target resource to be updated", + "title": "Target Resource" + }, + "description": "The target resource to be updated" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_process_data_process_data_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GoogleSheetValues" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/process-spreadsheet": { + "post": { + "summary": "Process Spreadsheet", + "description": "Process data to generate new ads or keywords based on the template", + "operationId": "process_spreadsheet_process_spreadsheet_post", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "description": "The user ID for which the data is requested", + "title": "User Id" + }, + "description": "The user ID for which the data is requested" + }, + { + "name": "template_spreadsheet_id", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "ID of the Google Sheet with the template data", + "title": "Template Spreadsheet Id" + }, + "description": "ID of the Google Sheet with the template data" + }, + { + "name": "template_sheet_title", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "The title of the sheet with the template data", + "title": "Template Sheet Title" + }, + "description": "The title of the sheet with the template data" + }, + { + "name": "new_campaign_spreadsheet_id", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "ID of the Google Sheet with the new campaign data", + "title": "New Campaign Spreadsheet Id" + }, + "description": "ID of the Google Sheet with the new campaign data" + }, + { + "name": "new_campaign_sheet_title", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "The title of the sheet with the new campaign data", + "title": "New Campaign Sheet Title" + }, + "description": "The title of the sheet with the new campaign data" + }, + { + "name": "target_resource", + "in": "query", + "required": true, + "schema": { + "enum": [ + "ad", + "keyword" + ], + "type": "string", + "description": "The target resource to be updated", + "title": "Target Resource" + }, + "description": "The target resource to be updated" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "Body_process_data_process_data_post": { + "properties": { + "template_sheet_values": { + "$ref": "#/components/schemas/GoogleSheetValues" + }, + "new_campaign_sheet_values": { + "$ref": "#/components/schemas/GoogleSheetValues" + } + }, + "type": "object", + "required": [ + "template_sheet_values", + "new_campaign_sheet_values" + ], + "title": "Body_process_data_process_data_post" + }, + "GoogleSheetValues": { + "properties": { + "values": { + "items": { + "items": {}, + "type": "array" + }, + "type": "array", + "title": "Values", + "description": "Values to be written to the Google Sheet." + } + }, + "type": "object", + "required": [ + "values" + ], + "title": "GoogleSheetValues" + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "type": "array", + "title": "Detail" + } + }, + "type": "object", + "title": "HTTPValidationError" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "type": "array", + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + }, + "type": "object", + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError" + } + } + } +} From aa1289111ef8473e7b85310640c3a641c660b3e6 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Mon, 1 Jul 2024 12:01:23 +0000 Subject: [PATCH 17/17] Uncomment postgres port forwarding in the devcontainer docker-compose files --- .devcontainer/python-3.10/docker-compose.yml | 4 ++-- .devcontainer/python-3.11/docker-compose.yml | 4 ++-- .devcontainer/python-3.12/docker-compose.yml | 4 ++-- .devcontainer/python-3.9/docker-compose.yml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.devcontainer/python-3.10/docker-compose.yml b/.devcontainer/python-3.10/docker-compose.yml index ce1a780..422be1d 100644 --- a/.devcontainer/python-3.10/docker-compose.yml +++ b/.devcontainer/python-3.10/docker-compose.yml @@ -20,8 +20,8 @@ services: POSTGRES_USER: admin POSTGRES_PASSWORD: password # pragma: allowlist secret POSTGRES_DB: google-sheets - # ports: - # - "${PORT_PREFIX}5432:5432" + ports: + - "${PORT_PREFIX}5432:5432" networks: - google-sheets-network diff --git a/.devcontainer/python-3.11/docker-compose.yml b/.devcontainer/python-3.11/docker-compose.yml index 35de7bc..8301642 100644 --- a/.devcontainer/python-3.11/docker-compose.yml +++ b/.devcontainer/python-3.11/docker-compose.yml @@ -20,8 +20,8 @@ services: POSTGRES_USER: admin POSTGRES_PASSWORD: password # pragma: allowlist secret POSTGRES_DB: google-sheets - # ports: - # - "${PORT_PREFIX}5432:5432" + ports: + - "${PORT_PREFIX}5432:5432" networks: - google-sheets-network diff --git a/.devcontainer/python-3.12/docker-compose.yml b/.devcontainer/python-3.12/docker-compose.yml index 82b25d6..875bcbe 100644 --- a/.devcontainer/python-3.12/docker-compose.yml +++ b/.devcontainer/python-3.12/docker-compose.yml @@ -20,8 +20,8 @@ services: POSTGRES_USER: admin POSTGRES_PASSWORD: password # pragma: allowlist secret POSTGRES_DB: google-sheets - # ports: - # - "${PORT_PREFIX}5432:5432" + ports: + - "${PORT_PREFIX}5432:5432" networks: - google-sheets-network diff --git a/.devcontainer/python-3.9/docker-compose.yml b/.devcontainer/python-3.9/docker-compose.yml index 342c8e7..f1a52b1 100644 --- a/.devcontainer/python-3.9/docker-compose.yml +++ b/.devcontainer/python-3.9/docker-compose.yml @@ -20,8 +20,8 @@ services: POSTGRES_USER: admin POSTGRES_PASSWORD: password # pragma: allowlist secret POSTGRES_DB: google-sheets - # ports: - # - "${PORT_PREFIX}5432:5432" + ports: + - "${PORT_PREFIX}5432:5432" networks: - google-sheets-network