From 59211dd16d70cfc677d8e983773af2507834b809 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matja=C5=BE=20Horvat?= <matjaz.horvat@gmail.com>
Date: Thu, 9 Nov 2023 03:18:30 +0100
Subject: [PATCH] Fix Project and ProjectLocale insights data (#3008)

Also included:
- Make created_at field visible in ActionLogAdmin
---
 pontoon/actionlog/admin.py                    |   1 +
 .../0015_fix_projectlocale_insights_data.py   | 111 ++++++++++++++++++
 pontoon/insights/tasks.py                     |   2 +-
 3 files changed, 113 insertions(+), 1 deletion(-)
 create mode 100644 pontoon/insights/migrations/0015_fix_projectlocale_insights_data.py

diff --git a/pontoon/actionlog/admin.py b/pontoon/actionlog/admin.py
index f3bd5b3410..775cbdcfe9 100644
--- a/pontoon/actionlog/admin.py
+++ b/pontoon/actionlog/admin.py
@@ -20,6 +20,7 @@ class ActionLogAdmin(admin.ModelAdmin):
         "entity",
         "translation",
     )
+    readonly_fields = ("created_at",)
 
 
 admin.site.register(ActionLog, ActionLogAdmin)
diff --git a/pontoon/insights/migrations/0015_fix_projectlocale_insights_data.py b/pontoon/insights/migrations/0015_fix_projectlocale_insights_data.py
new file mode 100644
index 0000000000..f0738a825c
--- /dev/null
+++ b/pontoon/insights/migrations/0015_fix_projectlocale_insights_data.py
@@ -0,0 +1,111 @@
+# Generated by Django 3.2.15 on 2023-10-30 14:15
+
+from django.db import migrations
+
+
+_activity_sql = """
+WITH filt AS (
+  WITH
+    sync AS (SELECT id FROM auth_user WHERE email = 'pontoon-sync@example.com')
+  SELECT
+    act.created_at::date AS created_at,
+    plx.id AS project_locale_id,
+    action_type = 'translation:created' AND cardinality(tra.machinery_sources) = 0 AS human_translation,
+    action_type = 'translation:created' AND cardinality(tra.machinery_sources) != 0 AS machinery_translation,
+    act.performed_by_id = sync.id AS is_sync,
+    act.action_type = 'translation:created' AND (tra.approved_date IS NULL OR tra.approved_date > tra.date) AS new_suggestion,
+    (
+      (action_type = 'translation:approved' AND act.performed_by_id = tra.user_id) OR
+      (action_type = 'translation:created' AND act.performed_by_id = tra.approved_user_id)
+    ) AS self_approved,
+    action_type = 'translation:approved' AND act.performed_by_id != tra.user_id AS peer_approved,
+    action_type = 'translation:rejected' AS rejected
+  FROM
+    sync,
+    actionlog_actionlog AS act,
+    base_translation AS tra,
+    base_entity AS ent,
+    base_resource AS res,
+    base_projectlocale AS plx
+  WHERE
+    act.translation_id = tra.id AND
+    tra.entity_id = ent.id AND
+    ent.resource_id = res.id AND
+    plx.locale_id = tra.locale_id AND plx.project_id = res.project_id
+), sums AS (
+  SELECT
+    created_at,
+    project_locale_id,
+    count(*) FILTER (WHERE filt.human_translation) AS human_translations,
+    count(*) FILTER (WHERE filt.machinery_translation) AS machinery_translations,
+    count(*) FILTER (WHERE NOT is_sync AND filt.new_suggestion) AS new_suggestions,
+    count(*) FILTER (WHERE NOT is_sync AND filt.self_approved) AS self_approved,
+    count(*) FILTER (WHERE NOT is_sync AND filt.peer_approved) AS peer_approved,
+    count(*) FILTER (WHERE NOT is_sync AND filt.rejected) AS rejected
+  FROM insights_projectlocaleinsightssnapshot JOIN filt USING (created_at, project_locale_id)
+  GROUP BY created_at, project_locale_id
+)
+UPDATE insights_projectlocaleinsightssnapshot AS ins
+SET
+  human_translations = sums.human_translations,
+  machinery_translations = sums.machinery_translations,
+  new_suggestions = sums.new_suggestions,
+  self_approved = sums.self_approved,
+  peer_approved = sums.peer_approved,
+  rejected = sums.rejected
+FROM sums
+WHERE
+  ins.created_at = sums.created_at + 1 AND
+  ins.project_locale_id = sums.project_locale_id;
+"""
+
+_reverse_activity_sql = """
+UPDATE insights_projectinsightssnapshot
+SET
+  human_translations = 0,
+  machinery_translations = 0,
+  new_suggestions = 0,
+  self_approved = 0,
+  peer_approved = 0,
+  rejected = 0;
+"""
+
+_new_sources_sql = """
+WITH ents AS (
+  SELECT ent.date_created::date AS created_at, project_id, count(*)
+  FROM
+    base_entity AS ent,
+    base_resource AS res,
+    base_project AS proj
+  WHERE
+    ent.resource_id = res.id AND
+    res.project_id = proj.id AND
+    NOT proj.disabled AND
+    NOT proj.system_project AND
+    proj.visibility = 'public'
+  GROUP BY ent.date_created::date, project_id
+)
+UPDATE insights_projectlocaleinsightssnapshot AS ins
+SET new_source_strings = ents.count
+FROM ents, base_projectlocale AS plx
+WHERE
+  ins.created_at = ents.created_at + 1 AND
+  ins.project_locale_id = plx.id AND
+  plx.project_id = ents.project_id;
+"""
+
+_reverse_new_sources_sql = """
+UPDATE insights_projectlocaleinsightssnapshot SET new_source_strings = 0;
+"""
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("insights", "0014_pretranslation_quality_data"),
+    ]
+
+    operations = [
+        migrations.RunSQL(sql=_activity_sql, reverse_sql=_reverse_activity_sql),
+        migrations.RunSQL(sql=_new_sources_sql, reverse_sql=_reverse_new_sources_sql),
+    ]
diff --git a/pontoon/insights/tasks.py b/pontoon/insights/tasks.py
index b1bb4f9ca7..4a97e51a1f 100644
--- a/pontoon/insights/tasks.py
+++ b/pontoon/insights/tasks.py
@@ -334,7 +334,7 @@ def get_project_locale_insights_snapshot(
         pretranslations_rejected,
         pretranslations_new,
     ) = get_activity_charts_data(
-        activities, locale=project_locale.project.id, project=project_locale.project.id
+        activities, locale=project_locale.locale.id, project=project_locale.project.id
     )
 
     return ProjectLocaleInsightsSnapshot(