Update captn templates (#78)

* Implement preprocess camaign data function

* Add Callouts, Search Network and max CPC to Campaigns sheet

* Refactoring

* wip languages

* Join based on language codes initial

* Language validation added

* Add target locations and languages

* Update columns

google_sheets/app.py

@@ -12,7 +12,12 @@
 from googleapiclient.errors import HttpError
 
 from . import __version__
-from .data_processing import process_data_f, validate_input_data, validate_output_data
+from .data_processing import (
+    process_campaign_data_f,
+    process_data_f,
+    validate_input_data,
+    validate_output_data,
+)
 from .db_helpers import get_db_connection
 from .google_api import (
     build_service,
@@ -140,6 +145,14 @@ async def login_callback(
     return RedirectResponse(redirect_uri)
 
 
+def _fill_rows_with_none(rows: List[List[Any]]) -> List[List[Any]]:
+    max_len = len(rows[0])
+    for i, row in enumerate(rows[1:]):
+        if len(row) < max_len:
+            rows[i + 1] += [None] * (max_len - len(row))
+    return rows
+
+
 @app.get("/get-sheet", description="Get data from a Google Sheet")
 async def get_sheet(
     user_id: Annotated[
@@ -164,6 +177,8 @@ async def get_sheet(
     if not values:
         return "No data found."
 
+    values = _fill_rows_with_none(values)
+
     return GoogleSheetValues(values=values)
 
 
@@ -314,8 +329,20 @@ async def get_all_sheet_titles(
     "Station To",
     "Final Url From",
     "Final Url To",
+    "Language Code",
+]
+
+MANDATORY_CAMPAIGN_TEMPLATE_COLUMNS = [
+    "Campaign Name",
+    "Language Code",
+    "Campaign Budget",
+    "Search Network",
+    "Google Search Network",
+    "Default max. CPC",
 ]
+
 MANDATORY_AD_TEMPLATE_COLUMNS = [
+    "Language Code",
     "Headline 1",
     "Headline 2",
     "Headline 3",
@@ -324,6 +351,7 @@ async def get_all_sheet_titles(
 ]
 
 MANDATORY_KEYWORD_TEMPLATE_COLUMNS = [
+    "Language Code",
     "Keyword",
     "Keyword Match Type",
     "Level",
@@ -339,6 +367,66 @@ def _validate_target_resource(target_resource: Optional[str]) -> None:
         )
 
 
+async def process_campaign_data(
+    template_sheet_values: GoogleSheetValues,
+    new_campaign_sheet_values: GoogleSheetValues,
+) -> GoogleSheetValues:
+    if (
+        len(template_sheet_values.values) < 2  # type: ignore
+        or len(new_campaign_sheet_values.values) < 2  # type: ignore
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Both template and new campaign data should have at least two rows (header and data).",
+        )
+    try:
+        template_df = pd.DataFrame(
+            template_sheet_values.values[1:],  # type: ignore
+            columns=template_sheet_values.values[0],  # type: ignore
+        )
+        new_campaign_df = pd.DataFrame(
+            new_campaign_sheet_values.values[1:],  # type: ignore
+            columns=new_campaign_sheet_values.values[0],  # type: ignore
+        )
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Invalid data format. Please provide data in the correct format: {e}",
+        ) from e
+
+    validation_error_msg = validate_input_data(
+        df=new_campaign_df,
+        mandatory_columns=NEW_CAMPAIGN_MANDATORY_COLUMNS,
+        name="new campaign",
+    )
+
+    validation_error_msg += validate_input_data(
+        df=template_df,
+        mandatory_columns=MANDATORY_CAMPAIGN_TEMPLATE_COLUMNS,
+        name="campaign template",
+    )
+
+    if validation_error_msg:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_msg
+        )
+
+    processed_df = process_campaign_data_f(
+        campaigns_template_df=template_df,
+        new_campaign_df=new_campaign_df,
+    )
+
+    validated_df = validate_output_data(
+        processed_df,
+        target_resource="campaign",  # type: ignore
+    )
+
+    issues_present = "Issues" in validated_df.columns
+    values = [validated_df.columns.tolist(), *validated_df.values.tolist()]
+
+    return GoogleSheetValues(values=values, issues_present=issues_present)
+
+
 async def process_data(
     template_sheet_values: GoogleSheetValues,
     new_campaign_sheet_values: GoogleSheetValues,
@@ -446,6 +534,34 @@ async def process_campaigns_and_ad_groups(
     return pd.merge(campaign_template_df, ad_group_template_df, how="cross")
 
 
+async def _create_and_update_sheet(
+    user_id: int,
+    new_campaign_spreadsheet_id: str,
+    processed_values: GoogleSheetValues,
+    target_resource: str,
+) -> str:
+    title = (
+        f"Captn - {target_resource.capitalize()}s {datetime.now():%Y-%m-%d %H:%M:%S}"  # type: ignore
+    )
+    await create_sheet(
+        user_id=user_id,
+        spreadsheet_id=new_campaign_spreadsheet_id,
+        title=title,
+    )
+    await update_sheet(
+        user_id=user_id,
+        spreadsheet_id=new_campaign_spreadsheet_id,
+        title=title,
+        sheet_values=processed_values,
+    )
+    response = f"Sheet with the name '{title}' has been created successfully.\n"
+    if processed_values.issues_present:
+        response += """But there are issues present in the data.
+Please check the 'Issues' column and correct the data accordingly.\n\n"""
+
+    return response
+
+
 @app.post(
     "/process-spreadsheet",
     description="Process data to generate new ads or keywords based on the template",
@@ -509,6 +625,19 @@ async def process_spreadsheet(
             ad_group_template_values=ad_group_template_values,
         )
 
+        drop_columns = [
+            "Campaign Budget",
+            "Search Network",
+            "Google Search Network",
+            "Default max. CPC",
+        ]
+        drop_columns += [
+            col
+            for col in merged_campaigns_ad_groups_df.columns
+            if col.lower().startswith("callout") or col.lower().startswith("sitelink")
+        ]
+        merged_campaigns_ad_groups_df.drop(columns=drop_columns, inplace=True)
+
     except Exception as e:
         raise HTTPException(
             status_code=status.HTTP_400_BAD_REQUEST,
@@ -532,32 +661,37 @@ async def process_spreadsheet(
 Please provide data in the correct format.""",
         )
 
-    response = ""
-    for template_values, target_resource in zip(
-        [ads_template_values, keywords_template_values], ["ad", "keyword"]
-    ):
-        processed_values = await process_data(
-            template_sheet_values=template_values,
+    try:
+        processed_values = await process_campaign_data(
+            template_sheet_values=campaign_template_values,
             new_campaign_sheet_values=new_campaign_values,
-            merged_campaigns_ad_groups_df=merged_campaigns_ad_groups_df,
-            target_resource=target_resource,
         )
 
-        title = f"Captn - {target_resource.capitalize()}s {datetime.now():%Y-%m-%d %H:%M:%S}"  # type: ignore
-        await create_sheet(
+        response = await _create_and_update_sheet(
             user_id=user_id,
-            spreadsheet_id=new_campaign_spreadsheet_id,
-            title=title,
+            new_campaign_spreadsheet_id=new_campaign_spreadsheet_id,  # type: ignore[arg-type]
+            processed_values=processed_values,
+            target_resource="campaign",
         )
-        await update_sheet(
-            user_id=user_id,
-            spreadsheet_id=new_campaign_spreadsheet_id,
-            title=title,
-            sheet_values=processed_values,
-        )
-        response += f"Sheet with the name '{title}' has been created successfully.\n"
-        if processed_values.issues_present:
-            response += """But there are issues present in the data.
-Please check the 'Issues' column and correct the data accordingly.\n\n"""
 
+        for template_values, target_resource in zip(
+            [ads_template_values, keywords_template_values], ["ad", "keyword"]
+        ):
+            processed_values = await process_data(
+                template_sheet_values=template_values,
+                new_campaign_sheet_values=new_campaign_values,
+                merged_campaigns_ad_groups_df=merged_campaigns_ad_groups_df,
+                target_resource=target_resource,
+            )
+
+            response += await _create_and_update_sheet(
+                user_id=user_id,
+                new_campaign_spreadsheet_id=new_campaign_spreadsheet_id,  # type: ignore[arg-type]
+                processed_values=processed_values,
+                target_resource=target_resource,
+            )
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)
+        ) from e
     return response

google_sheets/data_processing/__init__.py

@@ -1,3 +1,13 @@
-from .processing import process_data_f, validate_input_data, validate_output_data
+from .processing import (
+    process_campaign_data_f,
+    process_data_f,
+    validate_input_data,
+    validate_output_data,
+)
 
-__all__ = ["process_data_f", "validate_input_data", "validate_output_data"]
+__all__ = [
+    "process_campaign_data_f",
+    "process_data_f",
+    "validate_input_data",
+    "validate_output_data",
+]