Feat/dau reporting/add active users

sql_generators/glean_usage/__init__.py

@@ -18,6 +18,10 @@
     baseline_clients_first_seen,
     baseline_clients_last_seen,
     clients_last_seen_joined,
+    dau_reporting_clients_daily,
+    dau_reporting_clients_first_seen,
+    dau_reporting_clients_last_seen,
+    dau_reporting_active_users_aggregates,
     event_error_monitoring,
     event_flow_monitoring,
     event_monitoring_live,
@@ -43,6 +47,10 @@
     event_error_monitoring.EventErrorMonitoring(),
     event_flow_monitoring.EventFlowMonitoring(),
     events_stream.EventsStreamTable(),
+    dau_reporting_clients_daily.DauReportingClientsDailyTable(),
+    dau_reporting_clients_first_seen.DauReportingClientsFirstSeenTable(),
+    dau_reporting_clients_last_seen.DauReportingClientsLastSeenTable(),
+    dau_reporting_active_users_aggregates.DauReportingActiveUsersAggregatesTable(),
 ]
 
 
@@ -136,7 +144,7 @@ def get_tables(table_name="baseline_v1"):
         not in ConfigLoader.get("generate", "glean_usage", "skip_apps", fallback=[])
     ]
 
-    id_token=get_id_token()
+    id_token = get_id_token()
 
     # Prepare parameters so that generation of all Glean datasets can be done in parallel
 
@@ -151,7 +159,7 @@ def get_tables(table_name="baseline_v1"):
                 use_cloud_function=use_cloud_function,
                 app_info=app_info,
                 parallelism=parallelism,
-                id_token=id_token
+                id_token=id_token,
             ),
             baseline_table,
         )
@@ -169,7 +177,7 @@ def get_tables(table_name="baseline_v1"):
                 output_dir=output_dir,
                 use_cloud_function=use_cloud_function,
                 parallelism=parallelism,
-                id_token=id_token
+                id_token=id_token,
             ),
             info,
         )

sql_generators/glean_usage/common.py

@@ -151,6 +151,16 @@ def table_names_from_baseline(baseline_table, include_project_id=True):
         events_view=f"{prefix}.events",
         events_stream_table=f"{prefix}_derived.events_stream_v1",
         events_stream_view=f"{prefix}.events_stream",
+        dau_reporting_stable_table=f"{prefix}_stable.dau_reporting_v1",
+        dau_reporting_clients_daily_table=f"{prefix}_derived.dau_reporting_clients_daily_v1",
+        dau_reporting_clients_first_seen_table=f"{prefix}_derived.dau_reporting_clients_first_seen_v1",
+        dau_reporting_clients_last_seen_table=f"{prefix}_derived.dau_reporting_clients_last_seen_v1",
+        dau_reporting_active_users_aggregates_table=f"{prefix}_derived.dau_reporting_active_users_aggregates_v1",
+        dau_reporting_clients_daily_view=f"{prefix}.dau_reporting_clients_daily",
+        dau_reporting_clients_first_seen_view=f"{prefix}.dau_reporting_clients_first_seen",
+        dau_reporting_clients_last_seen_view=f"{prefix}.dau_reporting_clients_last_seen",
+        dau_reporting_active_users_view=f"{prefix}.dau_reporting_active_users",
+        dau_reporting_active_users_aggregates_view=f"{prefix}.dau_reporting_active_users_aggregates",
     )
 
 
@@ -234,7 +244,7 @@ def generate_per_app_id(
         use_cloud_function=True,
         app_info=[],
         parallelism=8,
-        id_token=None
+        id_token=None,
     ):
         """Generate the baseline table query per app_id."""
         if not self.per_app_id_enabled:
@@ -268,7 +278,7 @@ def generate_per_app_id(
             derived_dataset=derived_dataset,
             app_name=app_name,
             has_distribution_id=app_name in APPS_WITH_DISTRIBUTION_ID,
-            has_profile_group_id= app_name in APPS_WITH_PROFILE_GROUP_ID,
+            has_profile_group_id=app_name in APPS_WITH_PROFILE_GROUP_ID,
         )
 
         render_kwargs.update(self.custom_render_kwargs)
@@ -364,7 +374,7 @@ def generate_per_app(
         output_dir=None,
         use_cloud_function=True,
         parallelism=8,
-        id_token=None
+        id_token=None,
     ):
         """Generate the baseline table query per app_name."""
         if not self.per_app_enabled:

sql_generators/glean_usage/dau_reporting_active_users_aggregates.py

@@ -0,0 +1,17 @@
+"""Generating and run dau_reporting_active_users_aggregates queries for Glean apps."""
+
+from sql_generators.glean_usage.common import GleanTable
+
+TARGET_TABLE_ID = "dau_reporting_active_users_aggregates_v1"
+PREFIX = "dau_reporting_active_users_aggregates"
+
+
+class DauReportingActiveUsersAggregatesTable(GleanTable):
+    """Represents generated dau_reporting_active_users_aggregates table."""
+
+    def __init__(self):
+        """Initialize dau_reporting_active_users_aggregates table."""
+        GleanTable.__init__(self)
+        self.target_table_id = TARGET_TABLE_ID
+        self.prefix = PREFIX
+        self.base_table_name = "dau_reporting_v1"

sql_generators/glean_usage/dau_reporting_clients_daily.py

@@ -0,0 +1,17 @@
+"""Generating and run dau_reporting_clients_daily queries for Glean apps."""
+
+from sql_generators.glean_usage.common import GleanTable
+
+TARGET_TABLE_ID = "dau_reporting_clients_daily_v1"
+PREFIX = "dau_reporting_clients_daily"
+
+
+class DauReportingClientsDailyTable(GleanTable):
+    """Represents generated dau_reporting_clients_daily table."""
+
+    def __init__(self):
+        """Initialize dau_reporting_clients_daily table."""
+        GleanTable.__init__(self)
+        self.target_table_id = TARGET_TABLE_ID
+        self.prefix = PREFIX
+        self.base_table_name = "dau_reporting_v1"

sql_generators/glean_usage/dau_reporting_clients_first_seen.py

@@ -0,0 +1,17 @@
+"""Generating and run dau_reporting_clients_first_seen queries for Glean apps."""
+
+from sql_generators.glean_usage.common import GleanTable
+
+TARGET_TABLE_ID = "dau_reporting_clients_first_seen_v1"
+PREFIX = "dau_reporting_clients_first_seen"
+
+
+class DauReportingClientsFirstSeenTable(GleanTable):
+    """Represents generated dau_reporting_clients_first_seen table."""
+
+    def __init__(self):
+        """Initialize dau_reporting_clients_first_seen table."""
+        GleanTable.__init__(self)
+        self.target_table_id = TARGET_TABLE_ID
+        self.prefix = PREFIX
+        self.base_table_name = "dau_reporting_v1"

sql_generators/glean_usage/dau_reporting_clients_last_seen.py

@@ -0,0 +1,17 @@
+"""Generating and run dau_reporting_clients_last_seen queries for Glean apps."""
+
+from sql_generators.glean_usage.common import GleanTable
+
+TARGET_TABLE_ID = "dau_reporting_clients_last_seen_v1"
+PREFIX = "dau_reporting_clients_last_seen"
+
+
+class DauReportingClientsLastSeenTable(GleanTable):
+    """Represents generated dau_reporting_clients_last_seen table."""
+
+    def __init__(self):
+        """Initialize dau_reporting_clients_last_seen table."""
+        GleanTable.__init__(self)
+        self.target_table_id = TARGET_TABLE_ID
+        self.prefix = PREFIX
+        self.base_table_name = "dau_reporting_v1"

sql_generators/glean_usage/templates/cross_channel.view.sql

@@ -7,11 +7,11 @@ AS
 UNION ALL
 {% endif -%}
 {% if app_name == "fenix" -%}
-SELECT 
+SELECT
     "{{ dataset }}" AS normalized_app_id,
     * REPLACE(mozfun.norm.fenix_app_info("{{ dataset }}", app_build).channel AS normalized_channel),
 {% else -%}
-SELECT 
+SELECT
     "{{ dataset }}" AS normalized_app_id,
     * REPLACE("{{ channel }}" AS normalized_channel)
 {% endif -%}

sql_generators/glean_usage/templates/dau_reporting_active_users.metadata.yaml

@@ -0,0 +1,15 @@
+{{ header_yaml }}
+friendly_name: DAU Reporting Clients Last Seen
+description: |-
+  A daily client aggregation metrics for dau_reporting ping. Merges the computations for client first seen
+  and last seen metrics
+
+owners:
+  - [email protected]
+labels: {}
+bigquery: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:dataops-managed/taar
+  - workgroup:mozilla-confidential

sql_generators/glean_usage/templates/dau_reporting_active_users.schema.yaml

@@ -0,0 +1,137 @@
+fields:
+- mode: NULLABLE
+  name: submission_date
+  type: DATE
+  description: |
+    Logical date used for processing and paritioning.
+
+- mode: NULLABLE
+  name: usage_profile_id
+  type: STRING
+  description:
+
+- mode: NULLABLE
+  name: first_run_date
+  type: DATE
+  description: |
+    The date of the first run of the application.
+
+- mode: NULLABLE
+  name: normalized_channel
+  type: STRING
+  description: |
+    The channel the application is being distributed on.
+
+- mode: NULLABLE
+  name: normalized_os
+  type: STRING
+  description: |
+    The name of the operating system.
+
+- mode: NULLABLE
+  name: normalized_os_version
+  type: STRING
+  description: |
+    The user-visible version of the operating system (e.g. "1.2.3").
+    If the version detection fails, this metric gets set to Unknown.
+
+- mode: NULLABLE
+  name: locale
+  type: STRING
+  description: |
+    The locale of the application during initialization (e.g. "es-ES").
+    If the locale can't be determined on the system, the value is "und", to indicate "undetermined".
+
+- mode: NULLABLE
+  name: app_build
+  type: STRING
+  description: |
+    The build identifier generated by the CI system (e.g. "1234/A").
+    If the value was not provided through configuration, this metric gets set to Unknown.
+
+- mode: NULLABLE
+  name: app_display_version
+  type: STRING
+  description: |
+    The user visible version string (e.g. "1.0.3").
+    If the value was not provided through configuration, this metric gets set to Unknown.
+
+- mode: NULLABLE
+  name: distribution_id
+  type: STRING
+  description: |
+    A string containing the distribution identifier. This was used to identify installs
+    from Mozilla Online, but now also identifies partnership deal distributions.
+
+- mode: NULLABLE
+  name: is_active
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client was active.
+
+- mode: NULLABLE
+  name: first_seen_date
+  type: DATE
+  description: |
+    Logical date of when we observed the client for the first time in our warehouse.
+
+- mode: NULLABLE
+  name: days_seen_bits
+  type: INTEGER
+  description: |
+    Bit field shows on which of the last 28 days a client sent us the dau_reporting ping.
+
+- mode: NULLABLE
+  name: days_active_bits
+  type: INTEGER
+  description: |
+    Bit field shows on which of the last 28 days a client fulfilled the active criteria.
+
+- mode: NULLABLE
+  name: days_created_profile_bits
+  type: INTEGER
+  description: |
+    bit field indicating how many days lapsed since profile creation.
+
+- mode: NULLABLE
+  name: activity_segment
+  type: STRING
+  description: |
+    categorizing activity days into segments
+
+- mode: NULLABLE
+  name: is_dau
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client was active on the submission_date.
+
+- mode: NULLABLE
+  name: is_wau
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client was active on any of the 7 days prior to the submission_date.
+
+- mode: NULLABLE
+  name: is_mau
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client was active on any of the 28 days prior to the submission_date.
+
+- mode: NULLABLE
+  name: is_daily_user
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client sent the dau_reporting ping on the submission_date.
+
+- mode: NULLABLE
+  name: is_weekly_user
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client sent the dau_reporting ping on any of the 7 days prior to the submission_date.
+
+- mode: NULLABLE
+  name: is_monthly_user
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client sent the dau_reporting ping on any of the 28 days prior to the
+    submission_date.