Creating active_users and active_users_aggregates for dau_reporting ping

sql_generators/glean_usage/__init__.py

@@ -21,6 +21,7 @@
     dau_reporting_clients_daily,
     dau_reporting_clients_first_seen,
     dau_reporting_clients_last_seen,
+    dau_reporting_active_users_aggregates,
     event_error_monitoring,
     event_flow_monitoring,
     event_monitoring_live,
@@ -49,6 +50,7 @@
     dau_reporting_clients_daily.DauReportingClientsDailyTable(),
     dau_reporting_clients_first_seen.DauReportingClientsFirstSeenTable(),
     dau_reporting_clients_last_seen.DauReportingClientsLastSeenTable(),
+    dau_reporting_active_users_aggregates.DauReportingActiveUsersAggregatesTable(),
 ]
 
 

sql_generators/glean_usage/common.py

@@ -155,9 +155,12 @@ def table_names_from_baseline(baseline_table, include_project_id=True):
         dau_reporting_clients_daily_table=f"{prefix}_derived.dau_reporting_clients_daily_v1",
         dau_reporting_clients_first_seen_table=f"{prefix}_derived.dau_reporting_clients_first_seen_v1",
         dau_reporting_clients_last_seen_table=f"{prefix}_derived.dau_reporting_clients_last_seen_v1",
+        dau_reporting_active_users_aggregates_table=f"{prefix}_derived.dau_reporting_active_users_aggregates_v1",
         dau_reporting_clients_daily_view=f"{prefix}.dau_reporting_clients_daily",
         dau_reporting_clients_first_seen_view=f"{prefix}.dau_reporting_clients_first_seen",
         dau_reporting_clients_last_seen_view=f"{prefix}.dau_reporting_clients_last_seen",
+        dau_reporting_active_users_view=f"{prefix}.dau_reporting_active_users",
+        dau_reporting_active_users_aggregates_view=f"{prefix}.dau_reporting_active_users_aggregates",
     )
 
 

sql_generators/glean_usage/dau_reporting_active_users_aggregates.py

@@ -0,0 +1,17 @@
+"""Generating and run dau_reporting_active_users_aggregates queries for Glean apps."""
+
+from sql_generators.glean_usage.common import GleanTable
+
+TARGET_TABLE_ID = "dau_reporting_active_users_aggregates_v1"
+PREFIX = "dau_reporting_active_users_aggregates"
+
+
+class DauReportingActiveUsersAggregatesTable(GleanTable):
+    """Represents generated dau_reporting_active_users_aggregates table."""
+
+    def __init__(self):
+        """Initialize dau_reporting_active_users_aggregates table."""
+        GleanTable.__init__(self)
+        self.target_table_id = TARGET_TABLE_ID
+        self.prefix = PREFIX
+        self.base_table_name = "dau_reporting_v1"

sql_generators/glean_usage/templates/dau_reporting_active_users.metadata.yaml

@@ -0,0 +1,15 @@
+{{ header_yaml }}
+friendly_name: DAU Reporting Clients Last Seen
+description: |-
+  A daily client aggregation metrics for dau_reporting ping. Merges the computations for client first seen
+  and last seen metrics
+
+owners:
+  - [email protected]
+labels: {}
+bigquery: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:dataops-managed/taar
+  - workgroup:mozilla-confidential

sql_generators/glean_usage/templates/dau_reporting_active_users.schema.yaml

@@ -0,0 +1,137 @@
+fields:
+- mode: NULLABLE
+  name: submission_date
+  type: DATE
+  description: |
+    Logical date used for processing and paritioning.
+
+- mode: NULLABLE
+  name: usage_profile_id
+  type: STRING
+  description:
+
+- mode: NULLABLE
+  name: first_run_date
+  type: DATE
+  description: |
+    The date of the first run of the application.
+
+- mode: NULLABLE
+  name: normalized_channel
+  type: STRING
+  description: |
+    The channel the application is being distributed on.
+
+- mode: NULLABLE
+  name: normalized_os
+  type: STRING
+  description: |
+    The name of the operating system.
+
+- mode: NULLABLE
+  name: normalized_os_version
+  type: STRING
+  description: |
+    The user-visible version of the operating system (e.g. "1.2.3").
+    If the version detection fails, this metric gets set to Unknown.
+
+- mode: NULLABLE
+  name: locale
+  type: STRING
+  description: |
+    The locale of the application during initialization (e.g. "es-ES").
+    If the locale can't be determined on the system, the value is "und", to indicate "undetermined".
+
+- mode: NULLABLE
+  name: app_build
+  type: STRING
+  description: |
+    The build identifier generated by the CI system (e.g. "1234/A").
+    If the value was not provided through configuration, this metric gets set to Unknown.
+
+- mode: NULLABLE
+  name: app_display_version
+  type: STRING
+  description: |
+    The user visible version string (e.g. "1.0.3").
+    If the value was not provided through configuration, this metric gets set to Unknown.
+
+- mode: NULLABLE
+  name: distribution_id
+  type: STRING
+  description: |
+    A string containing the distribution identifier. This was used to identify installs
+    from Mozilla Online, but now also identifies partnership deal distributions.
+
+- mode: NULLABLE
+  name: is_active
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client was active.
+
+- mode: NULLABLE
+  name: first_seen_date
+  type: DATE
+  description: |
+    Logical date of when we observed the client for the first time in our warehouse.
+
+- mode: NULLABLE
+  name: days_seen_bits
+  type: INTEGER
+  description: |
+    Bit field shows on which of the last 28 days a client sent us the dau_reporting ping.
+
+- mode: NULLABLE
+  name: days_active_bits
+  type: INTEGER
+  description: |
+    Bit field shows on which of the last 28 days a client fulfilled the active criteria.
+
+- mode: NULLABLE
+  name: days_created_profile_bits
+  type: INTEGER
+  description: |
+    bit field indicating how many days lapsed since profile creation.
+
+- mode: NULLABLE
+  name: activity_segment
+  type: STRING
+  description: |
+    categorizing activity days into segments
+
+- mode: NULLABLE
+  name: is_dau
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client was active on the submission_date.
+
+- mode: NULLABLE
+  name: is_wau
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client was active on any of the 7 days prior to the submission_date.
+
+- mode: NULLABLE
+  name: is_mau
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client was active on any of the 28 days prior to the submission_date.
+
+- mode: NULLABLE
+  name: is_daily_user
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client sent the dau_reporting ping on the submission_date.
+
+- mode: NULLABLE
+  name: is_weekly_user
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client sent the dau_reporting ping on any of the 7 days prior to the submission_date.
+
+- mode: NULLABLE
+  name: is_monthly_user
+  type: BOOLEAN
+  description: |
+    A flag field indicating whether the specific client sent the dau_reporting ping on any of the 28 days prior to the
+    submission_date.

sql_generators/glean_usage/templates/dau_reporting_active_users.view.sql

@@ -0,0 +1,86 @@
+{{ header }}
+CREATE OR REPLACE VIEW
+  `{{ project_id }}.{{ dau_reporting_active_users_view }}`
+AS
+SELECT
+  submission_date,
+  usage_profile_id,
+  first_run_date,
+  normalized_channel,
+  normalized_os,
+  normalized_os_version,
+  locale,
+  app_build,
+  app_display_version,
+  distribution_id,
+  is_active,
+  first_seen_date,
+  days_seen_bits,
+  days_active_bits,
+  days_created_profile_bits,
+  CASE
+    WHEN BIT_COUNT(days_active_bits)
+      BETWEEN 1
+      AND 6
+      THEN 'infrequent_user'
+    WHEN BIT_COUNT(days_active_bits)
+      BETWEEN 7
+      AND 13
+      THEN 'casual_user'
+    WHEN BIT_COUNT(days_active_bits)
+      BETWEEN 14
+      AND 20
+      THEN 'regular_user'
+    WHEN BIT_COUNT(days_active_bits) >= 21
+      THEN 'core_user'
+    ELSE 'other'
+  END AS activity_segment,
+  IFNULL(mozfun.bits28.days_since_seen(days_active_bits) = 0, FALSE) AS is_dau,
+  IFNULL(mozfun.bits28.days_since_seen(days_active_bits) < 7, FALSE) AS is_wau,
+  IFNULL(mozfun.bits28.days_since_seen(days_active_bits) < 28, FALSE) AS is_mau,
+  IFNULL(mozfun.bits28.days_since_seen(days_seen_bits) = 0, FALSE) AS is_daily_user,
+  IFNULL(mozfun.bits28.days_since_seen(days_seen_bits) < 7, FALSE) AS is_weekly_user,
+  IFNULL(mozfun.bits28.days_since_seen(days_seen_bits) < 28, FALSE) AS is_monthly_user
+
+--
+-- TODO: uncomment once duration is added to the dau_reporting ping
+--
+-- -- Bit patterns capturing activity dates relative to the submission date.
+-- days_seen_session_start_bits,
+-- days_seen_session_end_bits,
+--
+
+-- -- TODO: verify if these fields are needed
+--   app_version,
+--   country,
+--   city,
+--   locale,
+--   os,
+--   windows_build_number,
+--   scalar_parent_browser_engagement_total_uri_count_normal_and_private_mode_sum,
+--   scalar_parent_browser_engagement_total_uri_count_sum,
+--   is_default_browser,
+--   isp_name,
+--   CASE
+--     WHEN isp_name = 'BrowserStack'
+--       THEN CONCAT('Firefox Desktop', ' ', isp_name)
+--     WHEN distribution_id = 'MozillaOnline'
+--       THEN CONCAT('Firefox Desktop', ' ', distribution_id)
+--     ELSE 'Firefox Desktop'
+--   END AS app_name,
+--   IF(
+--     LOWER(IFNULL(isp_name, '')) <> "browserstack"
+--     AND LOWER(IFNULL(distribution_id, '')) <> "mozillaonline",
+--     TRUE,
+--     FALSE
+--   ) AS is_desktop
+
+
+FROM
+  `{{ dau_reporting_clients_daily_table }}`
+LEFT JOIN
+  `{{ dau_reporting_clients_first_seen_table }}`
+  USING (usage_profile_id)
+LEFT JOIN
+  `{{ dau_reporting_clients_last_seen_table }}`
+  USING (usage_profile_id)

sql_generators/glean_usage/templates/dau_reporting_active_users_aggregates.metadata.yaml

@@ -0,0 +1,14 @@
+{{ header_yaml }}
+friendly_name: DAU Reporting Clients Last Seen
+description: |-
+  A daily aggregate of the dau_reporting ping representing user activity.
+
+owners:
+  - [email protected]
+labels: {}
+bigquery: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:dataops-managed/taar
+  - workgroup:mozilla-confidential

sql_generators/glean_usage/templates/dau_reporting_active_users_aggregates.view.sql

@@ -0,0 +1,8 @@
+{{ header }}
+CREATE OR REPLACE VIEW
+  `{{ project_id }}.{{ dau_reporting_active_users_aggregates_view }}`
+AS
+SELECT
+  *
+FROM
+  `{{ project_id }}.{{ dau_reporting_active_users_aggregates_table }}`

sql_generators/glean_usage/templates/dau_reporting_active_users_aggregates_v1.metadata.yaml

@@ -0,0 +1,22 @@
+{{ header_yaml }}
+friendly_name: DAU Reporting Clients Daily
+description: |-
+  A daily aggregate of the dau_reporting ping representing user activity.
+
+owners:
+  - [email protected]
+labels:
+  incremental: true
+  schedule: daily
+scheduling:
+  dag_name: bqetl_glean_usage
+  task_group: {{ app_name }}
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+  clustering:
+    fields:
+    - channel
+    - locale