From fab92c7e3522d326b7a416d832c04886fb0f055a Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Mon, 20 May 2024 14:08:48 -0300 Subject: [PATCH 1/4] Implement stats pixels --- .../DataBrokerProtectionPixelsHandler.swift | 6 +- .../Model/HistoryEvent.swift | 9 + ...taBrokerProfileQueryOperationManager.swift | 2 +- ...DataBrokerProtectionEngagementPixels.swift | 28 +- .../Pixels/DataBrokerProtectionPixels.swift | 55 ++- .../DataBrokerProtectionPixelsUtilities.swift | 43 +++ .../DataBrokerProtectionStatsPixels.swift | 364 +++++++++++++++++ .../DataBrokerProtectionQueueManager.swift | 3 + ...kerProfileQueryOperationManagerTests.swift | 19 + ...DataBrokerProtectionStatsPixelsTests.swift | 365 ++++++++++++++++++ .../DataBrokerProtectionTests/Mocks.swift | 37 ++ 11 files changed, 903 insertions(+), 28 deletions(-) create mode 100644 LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixelsUtilities.swift create mode 100644 LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift create mode 100644 LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionStatsPixelsTests.swift diff --git a/DuckDuckGo/DBP/DataBrokerProtectionPixelsHandler.swift b/DuckDuckGo/DBP/DataBrokerProtectionPixelsHandler.swift index 494dae251c..a073fa9aa5 100644 --- a/DuckDuckGo/DBP/DataBrokerProtectionPixelsHandler.swift +++ b/DuckDuckGo/DBP/DataBrokerProtectionPixelsHandler.swift @@ -93,7 +93,11 @@ public class DataBrokerProtectionPixelsHandler: EventMapping Bool { + switch type { + case .noMatchFound, .matchesFound(_): + return true + default: + return false + } + } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift index 72b5772a8b..4ec3876dab 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift @@ -139,11 +139,11 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager { stageCalculator.fireScanSuccess(matchesFound: extractedProfiles.count) let event = HistoryEvent(brokerId: brokerId, profileQueryId: profileQueryId, type: .matchesFound(count: extractedProfiles.count)) try database.add(event) + let extractedProfilesForBroker = try database.fetchExtractedProfiles(for: brokerId) for extractedProfile in extractedProfiles { // We check if the profile exists in the database. - let extractedProfilesForBroker = try database.fetchExtractedProfiles(for: brokerId) let doesProfileExistsInDatabase = extractedProfilesForBroker.contains { $0.identifier == extractedProfile.identifier } // If the profile exists we do not create a new opt-out operation diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionEngagementPixels.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionEngagementPixels.swift index e825e0eb33..918edd298f 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionEngagementPixels.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionEngagementPixels.swift @@ -92,14 +92,6 @@ final class DataBrokerProtectionEngagementPixelsUserDefaults: DataBrokerProtecti - MAU Pixel Last Sent 2024-03-19 */ final class DataBrokerProtectionEngagementPixels { - - enum ActiveUserFrequency: Int { - case daily = 1 - case weekly = 7 - case monthly = 28 - } - - private let calendar = Calendar.current private let database: DataBrokerProtectionRepository private let repository: DataBrokerProtectionEngagementPixelsRepository private let handler: EventMapping @@ -139,7 +131,7 @@ final class DataBrokerProtectionEngagementPixels { return true } - return shouldWeFirePixel(startDate: latestPixelFire, endDate: date, daysDifference: .daily) + return DataBrokerProtectionPixelsUtilities.shouldWeFirePixel(startDate: latestPixelFire, endDate: date, daysDifference: .daily) } private func shouldWeFireWeeklyPixel(date: Date) -> Bool { @@ -147,7 +139,7 @@ final class DataBrokerProtectionEngagementPixels { return true } - return shouldWeFirePixel(startDate: latestPixelFire, endDate: date, daysDifference: .weekly) + return DataBrokerProtectionPixelsUtilities.shouldWeFirePixel(startDate: latestPixelFire, endDate: date, daysDifference: .weekly) } private func shouldWeFireMonthlyPixel(date: Date) -> Bool { @@ -155,20 +147,6 @@ final class DataBrokerProtectionEngagementPixels { return true } - return shouldWeFirePixel(startDate: latestPixelFire, endDate: date, daysDifference: .monthly) - } - - private func shouldWeFirePixel(startDate: Date, endDate: Date, daysDifference: ActiveUserFrequency) -> Bool { - if let differenceBetweenDates = differenceBetweenDates(startDate: startDate, endDate: endDate) { - return differenceBetweenDates >= daysDifference.rawValue - } - - return false - } - - private func differenceBetweenDates(startDate: Date, endDate: Date) -> Int? { - let components = calendar.dateComponents([.day], from: startDate, to: endDate) - - return components.day + return DataBrokerProtectionPixelsUtilities.shouldWeFirePixel(startDate: latestPixelFire, endDate: date, daysDifference: .monthly) } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift index 5187485731..27acd866e3 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift @@ -70,6 +70,13 @@ public enum DataBrokerProtectionPixels { static let hasError = "has_error" static let brokerURL = "broker_url" static let sleepDuration = "sleep_duration" + static let numberOfRecordsFound = "num_found" + static let numberOfOptOutsInProgress = "num_inprogress" + static let numberOfSucessfulOptOuts = "num_optoutsuccess" + static let numberOfOptOutsFailure = "num_optoutfailure" + static let durationOfFirstOptOut = "duration_firstoptout" + static let numberOfNewRecordsFound = "num_new_found" + static let numberOfReappereances = "num_reappeared" } case error(error: DataBrokerProtectionError, dataBroker: String) @@ -171,6 +178,12 @@ public enum DataBrokerProtectionPixels { case entitlementCheckValid case entitlementCheckInvalid case entitlementCheckError + // Measure success/failure rate of Personal Information Removal Pixels + // https://app.asana.com/0/1204006570077678/1206889724879222/f + case globalMetricsWeeklyStats(profilesFound: Int, optOutsInProgress: Int, successfulOptOuts: Int, failedOptOuts: Int, durationOfFirstOptOut: Int, numberOfNewRecordsFound: Int) + case globalMetricsMonthlyStats(profilesFound: Int, optOutsInProgress: Int, successfulOptOuts: Int, failedOptOuts: Int, durationOfFirstOptOut: Int, numberOfNewRecordsFound: Int) + case dataBrokerMetricsWeeklyStats(dataBrokerURL: String, profilesFound: Int, optOutsInProgress: Int, successfulOptOuts: Int, failedOptOuts: Int, durationOfFirstOptOut: Int, numberOfNewRecordsFound: Int, numberOfReappereances: Int) + case dataBrokerMetricsMonthlyStats(dataBrokerURL: String, profilesFound: Int, optOutsInProgress: Int, successfulOptOuts: Int, failedOptOuts: Int, durationOfFirstOptOut: Int, numberOfNewRecordsFound: Int, numberOfReappereances: Int) } extension DataBrokerProtectionPixels: PixelKitEvent { @@ -281,6 +294,10 @@ extension DataBrokerProtectionPixels: PixelKitEvent { case .entitlementCheckValid: return "m_mac_dbp_macos_entitlement_valid" case .entitlementCheckInvalid: return "m_mac_dbp_macos_entitlement_invalid" case .entitlementCheckError: return "m_mac_dbp_macos_entitlement_error" + case .globalMetricsWeeklyStats: return "m_mac_dbp_weekly_stats" + case .globalMetricsMonthlyStats: return "m_mac_dbp_monthly_stats" + case .dataBrokerMetricsWeeklyStats: return "m_mac_dbp_databroker_weekly_stats" + case .dataBrokerMetricsMonthlyStats: return "m_mac_dbp_databroker_monthly_stats" } } @@ -419,6 +436,38 @@ extension DataBrokerProtectionPixels: PixelKitEvent { return [Consts.durationInMs: String(duration), Consts.hasError: hasError.description, Consts.brokerURL: brokerURL, Consts.sleepDuration: String(sleepDuration)] case .initialScanPreStartDuration(let duration): return [Consts.durationInMs: String(duration)] + case .globalMetricsWeeklyStats(let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound): + return [Consts.numberOfRecordsFound: String(profilesFound), + Consts.numberOfOptOutsInProgress: String(optOutsInProgress), + Consts.numberOfSucessfulOptOuts: String(successfulOptOuts), + Consts.numberOfOptOutsFailure: String(failedOptOuts), + Consts.durationOfFirstOptOut: String(durationOfFirstOptOut), + Consts.numberOfNewRecordsFound: String(numberOfNewRecordsFound)] + case .globalMetricsMonthlyStats(let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound): + return [Consts.numberOfRecordsFound: String(profilesFound), + Consts.numberOfOptOutsInProgress: String(optOutsInProgress), + Consts.numberOfSucessfulOptOuts: String(successfulOptOuts), + Consts.numberOfOptOutsFailure: String(failedOptOuts), + Consts.durationOfFirstOptOut: String(durationOfFirstOptOut), + Consts.numberOfNewRecordsFound: String(numberOfNewRecordsFound)] + case .dataBrokerMetricsWeeklyStats(let dataBrokerURL, let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound, let numberOfReappereances): + return [Consts.dataBrokerParamKey: dataBrokerURL, + Consts.numberOfRecordsFound: String(profilesFound), + Consts.numberOfOptOutsInProgress: String(optOutsInProgress), + Consts.numberOfSucessfulOptOuts: String(successfulOptOuts), + Consts.numberOfOptOutsFailure: String(failedOptOuts), + Consts.durationOfFirstOptOut: String(durationOfFirstOptOut), + Consts.numberOfNewRecordsFound: String(numberOfNewRecordsFound), + Consts.numberOfReappereances: String(numberOfReappereances)] + case .dataBrokerMetricsMonthlyStats(let dataBrokerURL, let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound, let numberOfReappereances): + return [Consts.dataBrokerParamKey: dataBrokerURL, + Consts.numberOfRecordsFound: String(profilesFound), + Consts.numberOfOptOutsInProgress: String(optOutsInProgress), + Consts.numberOfSucessfulOptOuts: String(successfulOptOuts), + Consts.numberOfOptOutsFailure: String(failedOptOuts), + Consts.durationOfFirstOptOut: String(durationOfFirstOptOut), + Consts.numberOfNewRecordsFound: String(numberOfNewRecordsFound), + Consts.numberOfReappereances: String(numberOfReappereances)] } } } @@ -498,7 +547,11 @@ public class DataBrokerProtectionPixelsHandler: EventMapping Bool { + if let differenceBetweenDates = differenceBetweenDates(startDate: startDate, endDate: endDate) { + return differenceBetweenDates >= daysDifference.rawValue + } + + return false + } + + static func differenceBetweenDates(startDate: Date, endDate: Date) -> Int? { + let components = calendar.dateComponents([.day], from: startDate, to: endDate) + + return components.day + } +} diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift new file mode 100644 index 0000000000..b411c8a8c0 --- /dev/null +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift @@ -0,0 +1,364 @@ +// +// DataBrokerProtectionStatsPixels.swift +// +// Copyright © 2024 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import Common +import BrowserServicesKit +import PixelKit + +protocol DataBrokerProtectionStatsPixelsRepository { + func markStatsWeeklyPixelDate() + func markStatsMonthlyPixelDate() + + func getLatestStatsWeeklyPixelDate() -> Date? + func getLatestStatsMonthlyPixelDate() -> Date? +} + +final class DataBrokerProtectionStatsPixelsUserDefaults: DataBrokerProtectionStatsPixelsRepository { + + enum Consts { + static let weeklyPixelKey = "macos.browser.data-broker-protection.statsWeeklyPixelKey" + static let monthlyPixelKey = "macos.browser.data-broker-protection.statsMonthlyPixelKey" + } + + private let userDefaults: UserDefaults + + init(userDefaults: UserDefaults = .dbp) { + self.userDefaults = userDefaults + } + + func markStatsWeeklyPixelDate() { + userDefaults.set(Date(), forKey: Consts.weeklyPixelKey) + } + + func markStatsMonthlyPixelDate() { + userDefaults.set(Date(), forKey: Consts.monthlyPixelKey) + } + + func getLatestStatsWeeklyPixelDate() -> Date? { + userDefaults.object(forKey: Consts.weeklyPixelKey) as? Date + } + + func getLatestStatsMonthlyPixelDate() -> Date? { + userDefaults.object(forKey: Consts.monthlyPixelKey) as? Date + } +} + +struct StatsByBroker { + let dataBrokerURL: String + let numberOfProfilesFound: Int + let numberOfOptOutsInProgress: Int + let numberOfSuccessfulOptOuts: Int + let numberOfFailureOptOuts: Int + let numberOfNewMatchesFound: Int + let numberOfReAppereances: Int + let durationOfFirstOptOut: Int + + var toWeeklyPixel: DataBrokerProtectionPixels { + return .dataBrokerMetricsWeeklyStats(dataBrokerURL: dataBrokerURL, + profilesFound: numberOfProfilesFound, + optOutsInProgress: numberOfOptOutsInProgress, + successfulOptOuts: numberOfSuccessfulOptOuts, + failedOptOuts: numberOfFailureOptOuts, + durationOfFirstOptOut: durationOfFirstOptOut, + numberOfNewRecordsFound: numberOfNewMatchesFound, + numberOfReappereances: numberOfReAppereances) + } + + var toMonthlyPixel: DataBrokerProtectionPixels { + return .dataBrokerMetricsMonthlyStats(dataBrokerURL: dataBrokerURL, + profilesFound: numberOfProfilesFound, + optOutsInProgress: numberOfOptOutsInProgress, + successfulOptOuts: numberOfSuccessfulOptOuts, + failedOptOuts: numberOfFailureOptOuts, + durationOfFirstOptOut: durationOfFirstOptOut, + numberOfNewRecordsFound: numberOfNewMatchesFound, + numberOfReappereances: numberOfReAppereances) + } +} + +extension Array where Element == StatsByBroker { + + func toWeeklyPixel(durationOfFirstOptOut: Int) -> DataBrokerProtectionPixels { + let numberOfGlobalProfilesFound = map { $0.numberOfProfilesFound }.reduce(0, +) + let numberOfGlobalOptOutsInProgress = map { $0.numberOfOptOutsInProgress }.reduce(0, +) + let numberOfGlobalSuccessfulOptOuts = map { $0.numberOfSuccessfulOptOuts }.reduce(0, +) + let numberOfGlobalFailureOptOuts = map { $0.numberOfFailureOptOuts }.reduce(0, +) + let numberOfGlobalNewMatchesFound = map { $0.numberOfNewMatchesFound }.reduce(0, +) + + return .globalMetricsWeeklyStats(profilesFound: numberOfGlobalProfilesFound, + optOutsInProgress: numberOfGlobalOptOutsInProgress, + successfulOptOuts: numberOfGlobalSuccessfulOptOuts, + failedOptOuts: numberOfGlobalFailureOptOuts, + durationOfFirstOptOut: durationOfFirstOptOut, + numberOfNewRecordsFound: numberOfGlobalNewMatchesFound) + } + + func toMonthlyPixel(durationOfFirstOptOut: Int) -> DataBrokerProtectionPixels { + let numberOfGlobalProfilesFound = map { $0.numberOfProfilesFound }.reduce(0, +) + let numberOfGlobalOptOutsInProgress = map { $0.numberOfOptOutsInProgress }.reduce(0, +) + let numberOfGlobalSuccessfulOptOuts = map { $0.numberOfSuccessfulOptOuts }.reduce(0, +) + let numberOfGlobalFailureOptOuts = map { $0.numberOfFailureOptOuts }.reduce(0, +) + let numberOfGlobalNewMatchesFound = map { $0.numberOfNewMatchesFound }.reduce(0, +) + + return .globalMetricsMonthlyStats(profilesFound: numberOfGlobalProfilesFound, + optOutsInProgress: numberOfGlobalOptOutsInProgress, + successfulOptOuts: numberOfGlobalSuccessfulOptOuts, + failedOptOuts: numberOfGlobalFailureOptOuts, + durationOfFirstOptOut: durationOfFirstOptOut, + numberOfNewRecordsFound: numberOfGlobalNewMatchesFound) + } +} + +final class DataBrokerProtectionStatsPixels { + private let database: DataBrokerProtectionRepository + private let handler: EventMapping + private let repository: DataBrokerProtectionStatsPixelsRepository + private let calendar = Calendar.current + + init(database: DataBrokerProtectionRepository, + handler: EventMapping, + repository: DataBrokerProtectionStatsPixelsRepository = DataBrokerProtectionStatsPixelsUserDefaults()) { + self.database = database + self.handler = handler + self.repository = repository + } + + func tryToFireStatsPixels() { + guard let brokerProfileQueryData = try? database.fetchAllBrokerProfileQueryData() else { + return + } + + let dateOfFirstScan = dateOfFirstScan(brokerProfileQueryData) + + if shouldFireWeeklyStats(dateOfFirstScan: dateOfFirstScan) { + firePixels(for: brokerProfileQueryData, frequency: .weekly) + repository.markStatsWeeklyPixelDate() + } + + if shouldFireMonthlyStats(dateOfFirstScan: dateOfFirstScan) { + firePixels(for: brokerProfileQueryData, frequency: .monthly) + repository.markStatsMonthlyPixelDate() + } + } + + private func shouldFireWeeklyStats(dateOfFirstScan: Date?) -> Bool { + // If no initial scan was done yet, we do not want to fire the pixel. + guard let dateOfFirstScan = dateOfFirstScan else { + return false + } + + if let lastWeeklyUpdateDate = repository.getLatestStatsWeeklyPixelDate() { + // If the last weekly was set we need to compare the date with it. + return DataBrokerProtectionPixelsUtilities.shouldWeFirePixel(startDate: lastWeeklyUpdateDate, endDate: Date(), daysDifference: .weekly) + } else { + // If the weekly update date was never set we need to check the first scan date. + return DataBrokerProtectionPixelsUtilities.shouldWeFirePixel(startDate: dateOfFirstScan, endDate: Date(), daysDifference: .weekly) + } + } + + private func shouldFireMonthlyStats(dateOfFirstScan: Date?) -> Bool { + // If no initial scan was done yet, we do not want to fire the pixel. + guard let dateOfFirstScan = dateOfFirstScan else { + return false + } + + if let lastMonthlyUpdateDate = repository.getLatestStatsMonthlyPixelDate() { + // If the last monthly was set we need to compare the date with it. + return DataBrokerProtectionPixelsUtilities.shouldWeFirePixel(startDate: lastMonthlyUpdateDate, endDate: Date(), daysDifference: .monthly) + } else { + // If the monthly update date was never set we need to check the first scan date. + return DataBrokerProtectionPixelsUtilities.shouldWeFirePixel(startDate: dateOfFirstScan, endDate: Date(), daysDifference: .monthly) + } + } + + private func firePixels(for brokerProfileQueryData: [BrokerProfileQueryData], frequency: Frequency) { + let statsByBroker = calculateStatsByBroker(brokerProfileQueryData) + + fireGlobalStats(statsByBroker, brokerProfileQueryData: brokerProfileQueryData, frequency: frequency) + fireStatsByBroker(statsByBroker, frequency: frequency) + } + + private func calculateStatsByBroker(_ brokerProfileQueryData: [BrokerProfileQueryData]) -> [StatsByBroker] { + let profileQueriesGroupedByBroker = Dictionary(grouping: brokerProfileQueryData, by: { $0.dataBroker }) + let statsByBroker = profileQueriesGroupedByBroker.map { (key: DataBroker, value: [BrokerProfileQueryData]) in + calculateByBroker(key, data: value) + } + + return statsByBroker + } + + private func fireGlobalStats(_ stats: [StatsByBroker], brokerProfileQueryData: [BrokerProfileQueryData], frequency: Frequency) { + // The duration for the global stats is calculated not taking into the account the broker. That's why we do not use one from the stats. + let durationOfFirstOptOut = calculateDurationOfFirstOptOut(brokerProfileQueryData) + + switch frequency { + case .weekly: + handler.fire(stats.toWeeklyPixel(durationOfFirstOptOut: durationOfFirstOptOut)) + case .monthly: + handler.fire(stats.toMonthlyPixel(durationOfFirstOptOut: durationOfFirstOptOut)) + default: () + } + } + + private func fireStatsByBroker(_ stats: [StatsByBroker], frequency: Frequency) { + for stat in stats { + switch frequency { + case .weekly: + handler.fire(stat.toWeeklyPixel) + case .monthly: + handler.fire(stat.toMonthlyPixel) + default: () + } + } + } + + /// internal for testing purposes + func calculateByBroker(_ broker: DataBroker, data: [BrokerProfileQueryData]) -> StatsByBroker { + let mirrorSitesSize = broker.mirrorSites.count + var numberOfProfilesFound = 0 // Number of unique matching profiles found since the beginning. + var numberOfOptOutsInProgress = 0 // Number of opt-outs in progress since the beginning. + var numberOfSuccessfulOptOuts = 0 // Number of successfull opt-outs since the beginning + var numberOfReAppearences = 0 // Number of records that were removed and came back + + for query in data { + for optOutData in query.optOutJobData { + if broker.performsOptOutWithinParent() { + // Path when the broker is a child site. + numberOfProfilesFound += 1 + if optOutData.historyEvents.contains(where: { $0.type == .optOutConfirmed }) { + numberOfSuccessfulOptOuts += 1 + } else { + numberOfOptOutsInProgress += 1 + } + } else { + // Path when the broker is a parent site. + // If we requested the opt-out successfully but we didn't remove it yet, it means it is in progress + numberOfProfilesFound += 1 + mirrorSitesSize + + if optOutData.historyEvents.contains(where: { $0.type == .optOutRequested }) && optOutData.extractedProfile.removedDate == nil { + numberOfOptOutsInProgress += 1 + mirrorSitesSize + } else if optOutData.extractedProfile.removedDate != nil { // If it the removed date is not nil, it means we removed it. + numberOfSuccessfulOptOuts += 1 + mirrorSitesSize + } + } + } + + numberOfReAppearences += calculateNumberOfReAppereances(query.scanJobData) + mirrorSitesSize + } + + let numberOfFailureOptOuts = numberOfProfilesFound - numberOfOptOutsInProgress - numberOfSuccessfulOptOuts + let numberOfNewMatchesFound = calculateNumberOfNewMatchesFound(data) + let durationOfFirstOptOut = calculateDurationOfFirstOptOut(data) + + return StatsByBroker(dataBrokerURL: broker.url, + numberOfProfilesFound: numberOfProfilesFound, + numberOfOptOutsInProgress: numberOfOptOutsInProgress, + numberOfSuccessfulOptOuts: numberOfSuccessfulOptOuts, + numberOfFailureOptOuts: numberOfFailureOptOuts, + numberOfNewMatchesFound: numberOfNewMatchesFound, + numberOfReAppereances: numberOfReAppearences, + durationOfFirstOptOut: durationOfFirstOptOut) + } + + /// Calculates number of new matches found on scans that were not initial scans. + /// + /// internal for testing purposes + func calculateNumberOfNewMatchesFound(_ brokerProfileQueryData: [BrokerProfileQueryData]) -> Int { + var numberOfNewMatches = 0 + + let brokerProfileQueryDataWithAMatch = brokerProfileQueryData.filter { !$0.extractedProfiles.isEmpty } + let profileQueriesGroupedByBroker = Dictionary(grouping: brokerProfileQueryDataWithAMatch, by: { $0.dataBroker }) + + profileQueriesGroupedByBroker.forEach { (key: DataBroker, value: [BrokerProfileQueryData]) in + let mirrorSitesCount = key.mirrorSites.count + + for query in value { + let matchesFoundEvents = query.scanJobData.historyEvents + .filter { $0.isMatchEvent() } + .sorted { $0.date < $1.date } + + matchesFoundEvents.enumerated().forEach { index, element in + if index > 0 && index < matchesFoundEvents.count - 1 { + let nextElement = matchesFoundEvents[index + 1] + numberOfNewMatches += max(nextElement.matchesFound() - element.matchesFound(), 0) + } + } + + if numberOfNewMatches > 0 { + numberOfNewMatches += mirrorSitesCount + } + } + } + + return numberOfNewMatches + } + + private func calculateNumberOfReAppereances(_ scan: ScanJobData) -> Int { + return scan.historyEvents.filter { $0.type == .reAppearence }.count + } + + /// Calculate the difference in days since the first scan and the first submitted opt-out for the list of brokerProfileQueryData. + /// The scan and the opt-out do not need to be for the same record. + /// If an opt-out wasn't submitted yet, we return 0. + /// + /// internal for testing purposes + func calculateDurationOfFirstOptOut(_ brokerProfileQueryData: [BrokerProfileQueryData]) -> Int { guard let dateOfFirstScan = dateOfFirstScan(brokerProfileQueryData), + let dateOfFirstSubmittedOptOut = dateOfFirstSubmittedOptOut(brokerProfileQueryData) else { + return 0 + } + + if dateOfFirstScan > dateOfFirstSubmittedOptOut { + return 0 + } + + guard let differenceInDays = DataBrokerProtectionPixelsUtilities.differenceBetweenDates(startDate: dateOfFirstScan, endDate: dateOfFirstSubmittedOptOut) else { + return 0 + } + + // If the difference in days is in hours, return 1. + if differenceInDays == 0 { + return 1 + } + + return differenceInDays + } + + /// Returns the date of the first scan + private func dateOfFirstScan(_ brokerProfileQueryData: [BrokerProfileQueryData]) -> Date? { + let allScanOperations = brokerProfileQueryData.map { $0.scanJobData } + let allScanHistoryEvents = allScanOperations.flatMap { $0.historyEvents } + let scanStartedEventsSortedByDate = allScanHistoryEvents + .filter { $0.type == .scanStarted } + .sorted { $0.date < $1.date } + + return scanStartedEventsSortedByDate.first?.date + } + + /// Returns the date of the first sumbitted opt-out + private func dateOfFirstSubmittedOptOut(_ brokerProfileQueryData: [BrokerProfileQueryData]) -> Date? { + let firstOptOutSubmittedEvent = brokerProfileQueryData + .flatMap { $0.optOutJobData } + .flatMap { $0.historyEvents } + .filter { $0.type == .optOutRequested } + .sorted { $0.date < $1.date } + .first + + return firstOptOutSubmittedEvent?.date + } +} diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Scheduler/DataBrokerProtectionQueueManager.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Scheduler/DataBrokerProtectionQueueManager.swift index 3567279d89..61f85122bb 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Scheduler/DataBrokerProtectionQueueManager.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Scheduler/DataBrokerProtectionQueueManager.swift @@ -250,11 +250,14 @@ private extension DefaultDataBrokerProtectionQueueManager { let engagementPixels = DataBrokerProtectionEngagementPixels(database: database, handler: pixelHandler) let eventPixels = DataBrokerProtectionEventPixels(database: database, handler: pixelHandler) + let statsPixels = DataBrokerProtectionStatsPixels(database: database, handler: pixelHandler) // This will fire the DAU/WAU/MAU pixels, engagementPixels.fireEngagementPixel() // This will try to fire the event weekly report pixels eventPixels.tryToFireWeeklyPixels() + // This will try to fire the stats pixels + statsPixels.tryToFireStatsPixels() } } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift index bd3aeafb71..b3d94d364e 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift @@ -1038,6 +1038,25 @@ extension DataBroker { ) ) } + + static func mockWith(mirroSites: [MirrorSite]) -> DataBroker { + DataBroker( + id: 1, + name: "Test broker", + url: "testbroker.com", + steps: [ + Step(type: .scan, actions: [Action]()), + Step(type: .optOut, actions: [Action]()) + ], + version: "1.0", + schedulingConfig: DataBrokerScheduleConfig( + retryError: 0, + confirmOptOutScan: 0, + maintenanceScan: 0 + ), + mirrorSites: mirroSites + ) + } } extension ExtractedProfile { diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionStatsPixelsTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionStatsPixelsTests.swift new file mode 100644 index 0000000000..62503b1fc1 --- /dev/null +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionStatsPixelsTests.swift @@ -0,0 +1,365 @@ +// +// DataBrokerProtectionStatsPixelsTests.swift +// +// Copyright © 2023 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest +import Foundation +@testable import DataBrokerProtection + +final class DataBrokerProtectionStatsPixelsTests: XCTestCase { + + func testNumberOfNewMatchesIsCalculatedCorrectly() { + let historyEvents: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 2)), + .init(brokerId: 1, profileQueryId: 1, type: .noMatchFound), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 1)), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 2)), + ] + let brokerProfileQueryData = BrokerProfileQueryData( + dataBroker: .mock, + profileQuery: .mock, + scanJobData: .mockWith(historyEvents: historyEvents), + optOutJobData: [.mock(with: .mockWithoutRemovedDate)]) + let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), + handler: MockDataBrokerProtectionPixelsHandler(), + repository: MockDataBrokerProtectionStatsPixelsRepository()) + + let result = sut.calculateNumberOfNewMatchesFound([brokerProfileQueryData]) + + XCTAssertEqual(result, 2) + } + + func testNumberOfNewMatchesIsCalculatedCorrectlyWithMirrorSites() { + let mirrorSites: [MirrorSite] = [ + .init(name: "Mirror #1", url: "url.com", addedAt: Date()), + .init(name: "Mirror #2", url: "url.com", addedAt: Date()) + ] + let historyEvents: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 2)), + .init(brokerId: 1, profileQueryId: 1, type: .noMatchFound), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 1)), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 2)), + ] + let brokerProfileQueryData = BrokerProfileQueryData( + dataBroker: .mockWith(mirroSites: mirrorSites), + profileQuery: .mock, + scanJobData: .mockWith(historyEvents: historyEvents), + optOutJobData: [.mock(with: .mockWithoutRemovedDate)]) + let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), + handler: MockDataBrokerProtectionPixelsHandler(), + repository: MockDataBrokerProtectionStatsPixelsRepository()) + + let result = sut.calculateNumberOfNewMatchesFound([brokerProfileQueryData]) + + XCTAssertEqual(result, 4) + } + + func testWhenDurationOfFirstOptOutIsLessThan24Hours_thenWeReturn1() { + let historyEventsForScan: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted, date: Date()), + ] + let historyEventsForOptOut: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .optOutRequested, date: Date()), + ] + let brokerProfileQueryData = BrokerProfileQueryData( + dataBroker: .mock, + profileQuery: .mock, + scanJobData: .mockWith(historyEvents: historyEventsForScan), + optOutJobData: [.mock(with: .mockWithoutRemovedDate, historyEvents: historyEventsForOptOut)]) + let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), + handler: MockDataBrokerProtectionPixelsHandler(), + repository: MockDataBrokerProtectionStatsPixelsRepository()) + + let result = sut.calculateDurationOfFirstOptOut([brokerProfileQueryData]) + + XCTAssertEqual(result, 1) + } + + func testWhenDateOfOptOutIsBeforeFirstScan_thenWeReturnZero() { + let historyEventsForScan: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted, date: Date()), + ] + let historyEventsForOptOut: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .optOutRequested, date: Date().yesterday!), + ] + let brokerProfileQueryData = BrokerProfileQueryData( + dataBroker: .mock, + profileQuery: .mock, + scanJobData: .mockWith(historyEvents: historyEventsForScan), + optOutJobData: [.mock(with: .mockWithoutRemovedDate, historyEvents: historyEventsForOptOut)]) + let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), + handler: MockDataBrokerProtectionPixelsHandler(), + repository: MockDataBrokerProtectionStatsPixelsRepository()) + + let result = sut.calculateDurationOfFirstOptOut([brokerProfileQueryData]) + + XCTAssertEqual(result, 0) + } + + func testWhenOptOutWasSubmitted_thenWeReturnCorrectNumberInDays() { + var dateComponents = DateComponents() + dateComponents.day = 3 + dateComponents.hour = 2 + let threeDaysAfterToday = Calendar.current.date(byAdding: dateComponents, to: Date())! + let historyEventsForScan: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted, date: Date()), + ] + let historyEventsForOptOut: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .optOutRequested, date: threeDaysAfterToday), + ] + let brokerProfileQueryData = BrokerProfileQueryData( + dataBroker: .mock, + profileQuery: .mock, + scanJobData: .mockWith(historyEvents: historyEventsForScan), + optOutJobData: [.mock(with: .mockWithoutRemovedDate, historyEvents: historyEventsForOptOut)]) + let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), + handler: MockDataBrokerProtectionPixelsHandler(), + repository: MockDataBrokerProtectionStatsPixelsRepository()) + + let result = sut.calculateDurationOfFirstOptOut([brokerProfileQueryData]) + + XCTAssertEqual(result, 3) + } + + /// This test data has the following parameters + /// - A broker that has two mirror sites + /// - Four matches found + /// - One match was removed + /// - Two matches are in progress of being removed (this means we submitted the opt-out) + /// - One match failed to submit an opt-out + /// - One re-appereance of an old match after it was removed + func testStatsByBroker_hasCorrectParams() { + let mirrorSites: [MirrorSite] = [ + .init(name: "Mirror #1", url: "url.com", addedAt: Date()), + .init(name: "Mirror #2", url: "url.com", addedAt: Date()) + ] + let broker: DataBroker = .mockWith(mirroSites: mirrorSites) + let historyEventsForFirstOptOutOperation: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .optOutStarted), + .init(brokerId: 1, profileQueryId: 1, type: .error(error: .unknown("Error"))), + .init(brokerId: 1, profileQueryId: 1, type: .optOutStarted), + .init(brokerId: 1, profileQueryId: 1, type: .error(error: .unknown("Error"))) + ] + let historyEventForOptOutWithSubmittedRequest: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .optOutStarted), + .init(brokerId: 1, profileQueryId: 1, type: .error(error: .unknown("Error"))), + .init(brokerId: 1, profileQueryId: 1, type: .optOutStarted), + .init(brokerId: 1, profileQueryId: 1, type: .optOutRequested) + ] + let historyEventsForScanOperation: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 3)), + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 2)), + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 3)), + .init(brokerId: 1, profileQueryId: 1, type: .reAppearence) + ] + let brokerProfileQueryData = BrokerProfileQueryData( + dataBroker: broker, + profileQuery: .mock, + scanJobData: .mockWith(historyEvents: historyEventsForScanOperation), + optOutJobData: [ + .mock(with: .mockWithoutRemovedDate, historyEvents: historyEventsForFirstOptOutOperation), + .mock(with: .mockWithoutRemovedDate, historyEvents: historyEventForOptOutWithSubmittedRequest), + .mock(with: .mockWithoutRemovedDate, historyEvents: historyEventForOptOutWithSubmittedRequest), + .mock(with: .mockWithRemovedDate, historyEvents: historyEventForOptOutWithSubmittedRequest), + ]) + let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), + handler: MockDataBrokerProtectionPixelsHandler(), + repository: MockDataBrokerProtectionStatsPixelsRepository()) + + let result = sut.calculateByBroker(broker, data: [brokerProfileQueryData]) + + XCTAssertEqual(result.numberOfProfilesFound, 12) + XCTAssertEqual(result.numberOfOptOutsInProgress, 6) + XCTAssertEqual(result.numberOfSuccessfulOptOuts, 3) + XCTAssertEqual(result.numberOfFailureOptOuts, 3) + XCTAssertEqual(result.numberOfNewMatchesFound, 3) + XCTAssertEqual(result.numberOfReAppereances, 3) + } + + /// This test data has the following parameters + /// - A broker that is a children site + /// - Three matches found + /// - One match was removed + /// - Two matches are in progress of being removed + func testStatsByBrokerForChildrenSite_hasCorrectParams() { + let broker: DataBroker = .mockWithParentOptOut + let historyEventsForFirstOptOutOperation: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .optOutConfirmed) + ] + let historyEventsForScanOperation: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 3)), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 2)), + ] + let brokerProfileQueryData = BrokerProfileQueryData( + dataBroker: broker, + profileQuery: .mock, + scanJobData: .mockWith(historyEvents: historyEventsForScanOperation), + optOutJobData: [ + .mock(with: .mockWithRemovedDate, historyEvents: historyEventsForFirstOptOutOperation), + .mock(with: .mockWithoutRemovedDate, historyEvents: [HistoryEvent]()), + .mock(with: .mockWithoutRemovedDate, historyEvents: [HistoryEvent]()) + ]) + let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), + handler: MockDataBrokerProtectionPixelsHandler(), + repository: MockDataBrokerProtectionStatsPixelsRepository()) + + let result = sut.calculateByBroker(broker, data: [brokerProfileQueryData]) + + XCTAssertEqual(result.numberOfProfilesFound, 3) + XCTAssertEqual(result.numberOfOptOutsInProgress, 2) + XCTAssertEqual(result.numberOfSuccessfulOptOuts, 1) + } + + func testWhenDateOfFirstScanIsNil_thenWeDoNotFireAnyPixel() { + let repository = MockDataBrokerProtectionStatsPixelsRepository() + let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), + handler: MockDataBrokerProtectionPixelsHandler(), + repository: repository) + + sut.tryToFireStatsPixels() + + XCTAssertFalse(repository.wasMarkStatsWeeklyPixelDateCalled) + XCTAssertFalse(repository.wasMarkStatsMonthlyPixelDateCalled) + } + + func testWhenLastWeeklyPixelIsNilAndAWeekDidntPassSinceInitialScan_thenWeDoNotFireWeeklyPixel() { + let database = MockDatabase() + let historyEventsForScanOperation: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted, date: Date().yesterday!), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 1), date: Date().yesterday!), + ] + database.brokerProfileQueryDataToReturn = [ + .init(dataBroker: .mock, profileQuery: .mock, scanJobData: .mockWith(historyEvents: historyEventsForScanOperation)) + ] + let repository = MockDataBrokerProtectionStatsPixelsRepository() + let sut = DataBrokerProtectionStatsPixels(database: database, + handler: MockDataBrokerProtectionPixelsHandler(), + repository: repository) + + sut.tryToFireStatsPixels() + + XCTAssertFalse(repository.wasMarkStatsWeeklyPixelDateCalled) + } + + func testWhenAWeekDidntPassSinceLastWeeklyPixelDate_thenWeDoNotFireWeeklyPixel() { + let database = MockDatabase() + let historyEventsForScanOperation: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted, date: Date().yesterday!), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 1), date: Date().yesterday!), + ] + database.brokerProfileQueryDataToReturn = [ + .init(dataBroker: .mock, profileQuery: .mock, scanJobData: .mockWith(historyEvents: historyEventsForScanOperation)) + ] + let repository = MockDataBrokerProtectionStatsPixelsRepository() + repository.latestStatsWeeklyPixelDate = Date().yesterday! + let sut = DataBrokerProtectionStatsPixels(database: database, + handler: MockDataBrokerProtectionPixelsHandler(), + repository: repository) + + sut.tryToFireStatsPixels() + + XCTAssertFalse(repository.wasMarkStatsWeeklyPixelDateCalled) + } + + func testWhenAWeekPassedSinceLastWeeklyPixelDate_thenWeFireWeeklyPixel() { + let eightDaysAgo = Calendar.current.date(byAdding: .day, value: -8, to: Date())! + let database = MockDatabase() + let historyEventsForScanOperation: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted, date: eightDaysAgo), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 1), date: eightDaysAgo), + ] + database.brokerProfileQueryDataToReturn = [ + .init(dataBroker: .mock, profileQuery: .mock, scanJobData: .mockWith(historyEvents: historyEventsForScanOperation)) + ] + let repository = MockDataBrokerProtectionStatsPixelsRepository() + repository.latestStatsWeeklyPixelDate = eightDaysAgo + let sut = DataBrokerProtectionStatsPixels(database: database, + handler: MockDataBrokerProtectionPixelsHandler(), + repository: repository) + + sut.tryToFireStatsPixels() + + XCTAssertTrue(repository.wasMarkStatsWeeklyPixelDateCalled) + } + + func testWhenLastMonthlyPixelIsNilAnd28DaysDidntPassSinceInitialScan_thenWeDoNotFireMonthlyPixel() { + let twentyDaysAgo = Calendar.current.date(byAdding: .day, value: -20, to: Date())! + let database = MockDatabase() + let historyEventsForScanOperation: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted, date: twentyDaysAgo), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 1), date: twentyDaysAgo), + ] + database.brokerProfileQueryDataToReturn = [ + .init(dataBroker: .mock, profileQuery: .mock, scanJobData: .mockWith(historyEvents: historyEventsForScanOperation)) + ] + let repository = MockDataBrokerProtectionStatsPixelsRepository() + let sut = DataBrokerProtectionStatsPixels(database: database, + handler: MockDataBrokerProtectionPixelsHandler(), + repository: repository) + + sut.tryToFireStatsPixels() + + XCTAssertFalse(repository.wasMarkStatsMonthlyPixelDateCalled) + } + + func testWhen28DaysDidntPassSinceLastMonthlyPixelDate_thenWeDoNotFireMonthlyPixel() { + let twentyDaysAgo = Calendar.current.date(byAdding: .day, value: -20, to: Date())! + let database = MockDatabase() + let historyEventsForScanOperation: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted, date: twentyDaysAgo), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 1), date: twentyDaysAgo), + ] + database.brokerProfileQueryDataToReturn = [ + .init(dataBroker: .mock, profileQuery: .mock, scanJobData: .mockWith(historyEvents: historyEventsForScanOperation)) + ] + let repository = MockDataBrokerProtectionStatsPixelsRepository() + repository.latestStatsMonthlyPixelDate = twentyDaysAgo + let sut = DataBrokerProtectionStatsPixels(database: database, + handler: MockDataBrokerProtectionPixelsHandler(), + repository: repository) + + sut.tryToFireStatsPixels() + + XCTAssertFalse(repository.wasMarkStatsMonthlyPixelDateCalled) + } + + func testWhen28DaysPassedSinceLastMonthlyPixelDate_thenWeFireMonthlyPixel() { + let thirtyDaysAgo = Calendar.current.date(byAdding: .day, value: -30, to: Date())! + let database = MockDatabase() + let historyEventsForScanOperation: [HistoryEvent] = [ + .init(brokerId: 1, profileQueryId: 1, type: .scanStarted, date: thirtyDaysAgo), + .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 1), date: thirtyDaysAgo), + ] + database.brokerProfileQueryDataToReturn = [ + .init(dataBroker: .mock, profileQuery: .mock, scanJobData: .mockWith(historyEvents: historyEventsForScanOperation)) + ] + let repository = MockDataBrokerProtectionStatsPixelsRepository() + repository.latestStatsMonthlyPixelDate = thirtyDaysAgo + let sut = DataBrokerProtectionStatsPixels(database: database, + handler: MockDataBrokerProtectionPixelsHandler(), + repository: repository) + + sut.tryToFireStatsPixels() + + XCTAssertTrue(repository.wasMarkStatsMonthlyPixelDateCalled) + } + +} diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift index fb2361e091..dd17954cd7 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift @@ -1008,6 +1008,13 @@ extension ScanJobData { } } +extension OptOutJobData { + static func mock(with extractedProfile: ExtractedProfile, + historyEvents: [HistoryEvent] = [HistoryEvent]()) -> OptOutJobData { + .init(brokerId: 1, profileQueryId: 1, historyEvents: historyEvents, extractedProfile: extractedProfile) + } +} + extension DataBroker { static func mock(withId id: Int64) -> DataBroker { @@ -1549,3 +1556,33 @@ extension SecureStorageError: Equatable { } } } + +final class MockDataBrokerProtectionStatsPixelsRepository: DataBrokerProtectionStatsPixelsRepository { + var wasMarkStatsWeeklyPixelDateCalled: Bool = false + var wasMarkStatsMonthlyPixelDateCalled: Bool = false + var latestStatsWeeklyPixelDate: Date? + var latestStatsMonthlyPixelDate: Date? + + func markStatsWeeklyPixelDate() { + wasMarkStatsWeeklyPixelDateCalled = true + } + + func markStatsMonthlyPixelDate() { + wasMarkStatsMonthlyPixelDateCalled = true + } + + func getLatestStatsWeeklyPixelDate() -> Date? { + return latestStatsWeeklyPixelDate + } + + func getLatestStatsMonthlyPixelDate() -> Date? { + return latestStatsMonthlyPixelDate + } + + func clear() { + wasMarkStatsWeeklyPixelDateCalled = false + wasMarkStatsMonthlyPixelDateCalled = false + latestStatsWeeklyPixelDate = nil + latestStatsMonthlyPixelDate = nil + } +} From 970c21de09c63d212af9d3080fe2e6cf0304ae27 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Thu, 23 May 2024 13:27:42 -0300 Subject: [PATCH 2/4] Address SwiftLint errors --- .../Sources/DataBrokerProtection/Model/HistoryEvent.swift | 2 +- .../Pixels/DataBrokerProtectionPixels.swift | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/HistoryEvent.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/HistoryEvent.swift index 16f43a18d9..c783ea4422 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/HistoryEvent.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/HistoryEvent.swift @@ -63,7 +63,7 @@ public struct HistoryEvent: Identifiable, Sendable { func isMatchEvent() -> Bool { switch type { - case .noMatchFound, .matchesFound(_): + case .noMatchFound, .matchesFound: return true default: return false diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift index 27acd866e3..d908a639de 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift @@ -440,14 +440,14 @@ extension DataBrokerProtectionPixels: PixelKitEvent { return [Consts.numberOfRecordsFound: String(profilesFound), Consts.numberOfOptOutsInProgress: String(optOutsInProgress), Consts.numberOfSucessfulOptOuts: String(successfulOptOuts), - Consts.numberOfOptOutsFailure: String(failedOptOuts), + Consts.numberOfOptOutsFailure: String(failedOptOuts), Consts.durationOfFirstOptOut: String(durationOfFirstOptOut), Consts.numberOfNewRecordsFound: String(numberOfNewRecordsFound)] case .globalMetricsMonthlyStats(let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound): return [Consts.numberOfRecordsFound: String(profilesFound), Consts.numberOfOptOutsInProgress: String(optOutsInProgress), Consts.numberOfSucessfulOptOuts: String(successfulOptOuts), - Consts.numberOfOptOutsFailure: String(failedOptOuts), + Consts.numberOfOptOutsFailure: String(failedOptOuts), Consts.durationOfFirstOptOut: String(durationOfFirstOptOut), Consts.numberOfNewRecordsFound: String(numberOfNewRecordsFound)] case .dataBrokerMetricsWeeklyStats(let dataBrokerURL, let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound, let numberOfReappereances): @@ -547,7 +547,7 @@ public class DataBrokerProtectionPixelsHandler: EventMapping Date: Thu, 23 May 2024 16:52:40 -0300 Subject: [PATCH 3/4] Fix tests --- .../DataBrokerProtectionStatsPixels.swift | 2 +- ...DataBrokerProtectionStatsPixelsTests.swift | 34 +++++++++++-------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift index b411c8a8c0..a265938d42 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift @@ -38,7 +38,7 @@ final class DataBrokerProtectionStatsPixelsUserDefaults: DataBrokerProtectionSta private let userDefaults: UserDefaults - init(userDefaults: UserDefaults = .dbp) { + init(userDefaults: UserDefaults = .standard) { self.userDefaults = userDefaults } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionStatsPixelsTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionStatsPixelsTests.swift index 62503b1fc1..f86a3af804 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionStatsPixelsTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionStatsPixelsTests.swift @@ -22,6 +22,12 @@ import Foundation final class DataBrokerProtectionStatsPixelsTests: XCTestCase { + private let handler = MockDataBrokerProtectionPixelsHandler() + + override func tearDown() { + handler.clear() + } + func testNumberOfNewMatchesIsCalculatedCorrectly() { let historyEvents: [HistoryEvent] = [ .init(brokerId: 1, profileQueryId: 1, type: .matchesFound(count: 2)), @@ -35,7 +41,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { scanJobData: .mockWith(historyEvents: historyEvents), optOutJobData: [.mock(with: .mockWithoutRemovedDate)]) let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: MockDataBrokerProtectionStatsPixelsRepository()) let result = sut.calculateNumberOfNewMatchesFound([brokerProfileQueryData]) @@ -60,7 +66,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { scanJobData: .mockWith(historyEvents: historyEvents), optOutJobData: [.mock(with: .mockWithoutRemovedDate)]) let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: MockDataBrokerProtectionStatsPixelsRepository()) let result = sut.calculateNumberOfNewMatchesFound([brokerProfileQueryData]) @@ -81,7 +87,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { scanJobData: .mockWith(historyEvents: historyEventsForScan), optOutJobData: [.mock(with: .mockWithoutRemovedDate, historyEvents: historyEventsForOptOut)]) let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: MockDataBrokerProtectionStatsPixelsRepository()) let result = sut.calculateDurationOfFirstOptOut([brokerProfileQueryData]) @@ -102,7 +108,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { scanJobData: .mockWith(historyEvents: historyEventsForScan), optOutJobData: [.mock(with: .mockWithoutRemovedDate, historyEvents: historyEventsForOptOut)]) let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: MockDataBrokerProtectionStatsPixelsRepository()) let result = sut.calculateDurationOfFirstOptOut([brokerProfileQueryData]) @@ -127,7 +133,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { scanJobData: .mockWith(historyEvents: historyEventsForScan), optOutJobData: [.mock(with: .mockWithoutRemovedDate, historyEvents: historyEventsForOptOut)]) let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: MockDataBrokerProtectionStatsPixelsRepository()) let result = sut.calculateDurationOfFirstOptOut([brokerProfileQueryData]) @@ -180,7 +186,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { .mock(with: .mockWithRemovedDate, historyEvents: historyEventForOptOutWithSubmittedRequest), ]) let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: MockDataBrokerProtectionStatsPixelsRepository()) let result = sut.calculateByBroker(broker, data: [brokerProfileQueryData]) @@ -218,7 +224,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { .mock(with: .mockWithoutRemovedDate, historyEvents: [HistoryEvent]()) ]) let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: MockDataBrokerProtectionStatsPixelsRepository()) let result = sut.calculateByBroker(broker, data: [brokerProfileQueryData]) @@ -231,7 +237,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { func testWhenDateOfFirstScanIsNil_thenWeDoNotFireAnyPixel() { let repository = MockDataBrokerProtectionStatsPixelsRepository() let sut = DataBrokerProtectionStatsPixels(database: MockDatabase(), - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: repository) sut.tryToFireStatsPixels() @@ -251,7 +257,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { ] let repository = MockDataBrokerProtectionStatsPixelsRepository() let sut = DataBrokerProtectionStatsPixels(database: database, - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: repository) sut.tryToFireStatsPixels() @@ -271,7 +277,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { let repository = MockDataBrokerProtectionStatsPixelsRepository() repository.latestStatsWeeklyPixelDate = Date().yesterday! let sut = DataBrokerProtectionStatsPixels(database: database, - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: repository) sut.tryToFireStatsPixels() @@ -292,7 +298,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { let repository = MockDataBrokerProtectionStatsPixelsRepository() repository.latestStatsWeeklyPixelDate = eightDaysAgo let sut = DataBrokerProtectionStatsPixels(database: database, - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: repository) sut.tryToFireStatsPixels() @@ -312,7 +318,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { ] let repository = MockDataBrokerProtectionStatsPixelsRepository() let sut = DataBrokerProtectionStatsPixels(database: database, - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: repository) sut.tryToFireStatsPixels() @@ -333,7 +339,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { let repository = MockDataBrokerProtectionStatsPixelsRepository() repository.latestStatsMonthlyPixelDate = twentyDaysAgo let sut = DataBrokerProtectionStatsPixels(database: database, - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: repository) sut.tryToFireStatsPixels() @@ -354,7 +360,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { let repository = MockDataBrokerProtectionStatsPixelsRepository() repository.latestStatsMonthlyPixelDate = thirtyDaysAgo let sut = DataBrokerProtectionStatsPixels(database: database, - handler: MockDataBrokerProtectionPixelsHandler(), + handler: handler, repository: repository) sut.tryToFireStatsPixels() From c55c7d9d1ca32821c513bd950f357c379f89ae3b Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Sun, 9 Jun 2024 12:30:20 -0300 Subject: [PATCH 4/4] Address feedback --- .../Model/DataBroker.swift | 8 +++ .../Pixels/DataBrokerProtectionPixels.swift | 50 +++++++------------ .../DataBrokerProtectionPixelsUtilities.swift | 4 +- .../DataBrokerProtectionStatsPixels.swift | 50 ++++++++++++------- ...DataBrokerProtectionStatsPixelsTests.swift | 16 +++--- 5 files changed, 67 insertions(+), 61 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift index 41a5293846..44e62f952d 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift @@ -67,6 +67,14 @@ struct MirrorSite: Codable, Sendable { removedAt = try? container.decode(Date.self, forKey: .removedAt) } + + func wasRemoved(since: Date = Date()) -> Bool { + guard let removedAt = self.removedAt else { + return false + } + + return removedAt < since + } } public enum DataBrokerHierarchy: Int { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift index d908a639de..7d2f3fcd30 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixels.swift @@ -436,38 +436,24 @@ extension DataBrokerProtectionPixels: PixelKitEvent { return [Consts.durationInMs: String(duration), Consts.hasError: hasError.description, Consts.brokerURL: brokerURL, Consts.sleepDuration: String(sleepDuration)] case .initialScanPreStartDuration(let duration): return [Consts.durationInMs: String(duration)] - case .globalMetricsWeeklyStats(let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound): - return [Consts.numberOfRecordsFound: String(profilesFound), - Consts.numberOfOptOutsInProgress: String(optOutsInProgress), - Consts.numberOfSucessfulOptOuts: String(successfulOptOuts), - Consts.numberOfOptOutsFailure: String(failedOptOuts), - Consts.durationOfFirstOptOut: String(durationOfFirstOptOut), - Consts.numberOfNewRecordsFound: String(numberOfNewRecordsFound)] - case .globalMetricsMonthlyStats(let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound): - return [Consts.numberOfRecordsFound: String(profilesFound), - Consts.numberOfOptOutsInProgress: String(optOutsInProgress), - Consts.numberOfSucessfulOptOuts: String(successfulOptOuts), - Consts.numberOfOptOutsFailure: String(failedOptOuts), - Consts.durationOfFirstOptOut: String(durationOfFirstOptOut), - Consts.numberOfNewRecordsFound: String(numberOfNewRecordsFound)] - case .dataBrokerMetricsWeeklyStats(let dataBrokerURL, let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound, let numberOfReappereances): - return [Consts.dataBrokerParamKey: dataBrokerURL, - Consts.numberOfRecordsFound: String(profilesFound), - Consts.numberOfOptOutsInProgress: String(optOutsInProgress), - Consts.numberOfSucessfulOptOuts: String(successfulOptOuts), - Consts.numberOfOptOutsFailure: String(failedOptOuts), - Consts.durationOfFirstOptOut: String(durationOfFirstOptOut), - Consts.numberOfNewRecordsFound: String(numberOfNewRecordsFound), - Consts.numberOfReappereances: String(numberOfReappereances)] - case .dataBrokerMetricsMonthlyStats(let dataBrokerURL, let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound, let numberOfReappereances): - return [Consts.dataBrokerParamKey: dataBrokerURL, - Consts.numberOfRecordsFound: String(profilesFound), - Consts.numberOfOptOutsInProgress: String(optOutsInProgress), - Consts.numberOfSucessfulOptOuts: String(successfulOptOuts), - Consts.numberOfOptOutsFailure: String(failedOptOuts), - Consts.durationOfFirstOptOut: String(durationOfFirstOptOut), - Consts.numberOfNewRecordsFound: String(numberOfNewRecordsFound), - Consts.numberOfReappereances: String(numberOfReappereances)] + case .globalMetricsWeeklyStats(let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound), + .globalMetricsMonthlyStats(let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound): + return [Consts.numberOfRecordsFound: String(profilesFound), + Consts.numberOfOptOutsInProgress: String(optOutsInProgress), + Consts.numberOfSucessfulOptOuts: String(successfulOptOuts), + Consts.numberOfOptOutsFailure: String(failedOptOuts), + Consts.durationOfFirstOptOut: String(durationOfFirstOptOut), + Consts.numberOfNewRecordsFound: String(numberOfNewRecordsFound)] + case .dataBrokerMetricsWeeklyStats(let dataBrokerURL, let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound, let numberOfReappereances), + .dataBrokerMetricsMonthlyStats(let dataBrokerURL, let profilesFound, let optOutsInProgress, let successfulOptOuts, let failedOptOuts, let durationOfFirstOptOut, let numberOfNewRecordsFound, let numberOfReappereances): + return [Consts.dataBrokerParamKey: dataBrokerURL, + Consts.numberOfRecordsFound: String(profilesFound), + Consts.numberOfOptOutsInProgress: String(optOutsInProgress), + Consts.numberOfSucessfulOptOuts: String(successfulOptOuts), + Consts.numberOfOptOutsFailure: String(failedOptOuts), + Consts.durationOfFirstOptOut: String(durationOfFirstOptOut), + Consts.numberOfNewRecordsFound: String(numberOfNewRecordsFound), + Consts.numberOfReappereances: String(numberOfReappereances)] } } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixelsUtilities.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixelsUtilities.swift index 5b81da2297..c3b9f3474c 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixelsUtilities.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionPixelsUtilities.swift @@ -28,14 +28,14 @@ final class DataBrokerProtectionPixelsUtilities { private static let calendar = Calendar.current static func shouldWeFirePixel(startDate: Date, endDate: Date, daysDifference: Frequency) -> Bool { - if let differenceBetweenDates = differenceBetweenDates(startDate: startDate, endDate: endDate) { + if let differenceBetweenDates = numberOfDaysFrom(startDate: startDate, endDate: endDate) { return differenceBetweenDates >= daysDifference.rawValue } return false } - static func differenceBetweenDates(startDate: Date, endDate: Date) -> Int? { + static func numberOfDaysFrom(startDate: Date, endDate: Date) -> Int? { let components = calendar.dateComponents([.day], from: startDate, to: endDate) return components.day diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift index a265938d42..e14bbddcec 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift @@ -147,12 +147,16 @@ final class DataBrokerProtectionStatsPixels { let dateOfFirstScan = dateOfFirstScan(brokerProfileQueryData) if shouldFireWeeklyStats(dateOfFirstScan: dateOfFirstScan) { - firePixels(for: brokerProfileQueryData, frequency: .weekly) + firePixels(for: brokerProfileQueryData, + frequency: .weekly, + dateSinceLastSubmission: repository.getLatestStatsWeeklyPixelDate()) repository.markStatsWeeklyPixelDate() } if shouldFireMonthlyStats(dateOfFirstScan: dateOfFirstScan) { - firePixels(for: brokerProfileQueryData, frequency: .monthly) + firePixels(for: brokerProfileQueryData, + frequency: .monthly, + dateSinceLastSubmission: repository.getLatestStatsMonthlyPixelDate()) repository.markStatsMonthlyPixelDate() } } @@ -187,17 +191,17 @@ final class DataBrokerProtectionStatsPixels { } } - private func firePixels(for brokerProfileQueryData: [BrokerProfileQueryData], frequency: Frequency) { - let statsByBroker = calculateStatsByBroker(brokerProfileQueryData) + private func firePixels(for brokerProfileQueryData: [BrokerProfileQueryData], frequency: Frequency, dateSinceLastSubmission: Date? = nil) { + let statsByBroker = calculateStatsByBroker(brokerProfileQueryData, dateSinceLastSubmission: dateSinceLastSubmission) fireGlobalStats(statsByBroker, brokerProfileQueryData: brokerProfileQueryData, frequency: frequency) fireStatsByBroker(statsByBroker, frequency: frequency) } - private func calculateStatsByBroker(_ brokerProfileQueryData: [BrokerProfileQueryData]) -> [StatsByBroker] { + private func calculateStatsByBroker(_ brokerProfileQueryData: [BrokerProfileQueryData], dateSinceLastSubmission: Date? = nil) -> [StatsByBroker] { let profileQueriesGroupedByBroker = Dictionary(grouping: brokerProfileQueryData, by: { $0.dataBroker }) let statsByBroker = profileQueriesGroupedByBroker.map { (key: DataBroker, value: [BrokerProfileQueryData]) in - calculateByBroker(key, data: value) + calculateByBroker(key, data: value, dateSinceLastSubmission: dateSinceLastSubmission) } return statsByBroker @@ -229,8 +233,8 @@ final class DataBrokerProtectionStatsPixels { } /// internal for testing purposes - func calculateByBroker(_ broker: DataBroker, data: [BrokerProfileQueryData]) -> StatsByBroker { - let mirrorSitesSize = broker.mirrorSites.count + func calculateByBroker(_ broker: DataBroker, data: [BrokerProfileQueryData], dateSinceLastSubmission: Date? = nil) -> StatsByBroker { + let mirrorSitesSize = broker.mirrorSites.filter { !$0.wasRemoved() }.count var numberOfProfilesFound = 0 // Number of unique matching profiles found since the beginning. var numberOfOptOutsInProgress = 0 // Number of opt-outs in progress since the beginning. var numberOfSuccessfulOptOuts = 0 // Number of successfull opt-outs since the beginning @@ -264,7 +268,7 @@ final class DataBrokerProtectionStatsPixels { let numberOfFailureOptOuts = numberOfProfilesFound - numberOfOptOutsInProgress - numberOfSuccessfulOptOuts let numberOfNewMatchesFound = calculateNumberOfNewMatchesFound(data) - let durationOfFirstOptOut = calculateDurationOfFirstOptOut(data) + let durationOfFirstOptOut = calculateDurationOfFirstOptOut(data, from: dateSinceLastSubmission) return StatsByBroker(dataBrokerURL: broker.url, numberOfProfilesFound: numberOfProfilesFound, @@ -286,7 +290,7 @@ final class DataBrokerProtectionStatsPixels { let profileQueriesGroupedByBroker = Dictionary(grouping: brokerProfileQueryDataWithAMatch, by: { $0.dataBroker }) profileQueriesGroupedByBroker.forEach { (key: DataBroker, value: [BrokerProfileQueryData]) in - let mirrorSitesCount = key.mirrorSites.count + let mirrorSitesCount = key.mirrorSites.filter { !$0.wasRemoved() }.count for query in value { let matchesFoundEvents = query.scanJobData.historyEvents @@ -318,7 +322,8 @@ final class DataBrokerProtectionStatsPixels { /// If an opt-out wasn't submitted yet, we return 0. /// /// internal for testing purposes - func calculateDurationOfFirstOptOut(_ brokerProfileQueryData: [BrokerProfileQueryData]) -> Int { guard let dateOfFirstScan = dateOfFirstScan(brokerProfileQueryData), + func calculateDurationOfFirstOptOut(_ brokerProfileQueryData: [BrokerProfileQueryData], from: Date? = nil) -> Int { + guard let dateOfFirstScan = dateOfFirstScan(brokerProfileQueryData), let dateOfFirstSubmittedOptOut = dateOfFirstSubmittedOptOut(brokerProfileQueryData) else { return 0 } @@ -327,7 +332,7 @@ final class DataBrokerProtectionStatsPixels { return 0 } - guard let differenceInDays = DataBrokerProtectionPixelsUtilities.differenceBetweenDates(startDate: dateOfFirstScan, endDate: dateOfFirstSubmittedOptOut) else { + guard let differenceInDays = DataBrokerProtectionPixelsUtilities.numberOfDaysFrom(startDate: dateOfFirstScan, endDate: dateOfFirstSubmittedOptOut) else { return 0 } @@ -339,26 +344,33 @@ final class DataBrokerProtectionStatsPixels { return differenceInDays } - /// Returns the date of the first scan - private func dateOfFirstScan(_ brokerProfileQueryData: [BrokerProfileQueryData]) -> Date? { + /// Returns the date of the first scan since the beginning if not from Date is provided + private func dateOfFirstScan(_ brokerProfileQueryData: [BrokerProfileQueryData], from: Date? = nil) -> Date? { let allScanOperations = brokerProfileQueryData.map { $0.scanJobData } let allScanHistoryEvents = allScanOperations.flatMap { $0.historyEvents } let scanStartedEventsSortedByDate = allScanHistoryEvents .filter { $0.type == .scanStarted } .sorted { $0.date < $1.date } - return scanStartedEventsSortedByDate.first?.date + if let from = from { + return scanStartedEventsSortedByDate.filter { from < $0.date }.first?.date + } else { + return scanStartedEventsSortedByDate.first?.date + } } - /// Returns the date of the first sumbitted opt-out - private func dateOfFirstSubmittedOptOut(_ brokerProfileQueryData: [BrokerProfileQueryData]) -> Date? { + /// Returns the date of the first sumbitted opt-out. If no from date is provided, we return it from the beginning. + private func dateOfFirstSubmittedOptOut(_ brokerProfileQueryData: [BrokerProfileQueryData], from: Date? = nil) -> Date? { let firstOptOutSubmittedEvent = brokerProfileQueryData .flatMap { $0.optOutJobData } .flatMap { $0.historyEvents } .filter { $0.type == .optOutRequested } .sorted { $0.date < $1.date } - .first - return firstOptOutSubmittedEvent?.date + if let from = from { + return firstOptOutSubmittedEvent.filter { from < $0.date }.first?.date + } else { + return firstOptOutSubmittedEvent.first?.date + } } } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionStatsPixelsTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionStatsPixelsTests.swift index f86a3af804..cc8298bd71 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionStatsPixelsTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionStatsPixelsTests.swift @@ -142,7 +142,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { } /// This test data has the following parameters - /// - A broker that has two mirror sites + /// - A broker that has two mirror sites but one was removed /// - Four matches found /// - One match was removed /// - Two matches are in progress of being removed (this means we submitted the opt-out) @@ -151,7 +151,7 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { func testStatsByBroker_hasCorrectParams() { let mirrorSites: [MirrorSite] = [ .init(name: "Mirror #1", url: "url.com", addedAt: Date()), - .init(name: "Mirror #2", url: "url.com", addedAt: Date()) + .init(name: "Mirror #2", url: "url.com", addedAt: Date(), removedAt: Date().yesterday) ] let broker: DataBroker = .mockWith(mirroSites: mirrorSites) let historyEventsForFirstOptOutOperation: [HistoryEvent] = [ @@ -191,12 +191,12 @@ final class DataBrokerProtectionStatsPixelsTests: XCTestCase { let result = sut.calculateByBroker(broker, data: [brokerProfileQueryData]) - XCTAssertEqual(result.numberOfProfilesFound, 12) - XCTAssertEqual(result.numberOfOptOutsInProgress, 6) - XCTAssertEqual(result.numberOfSuccessfulOptOuts, 3) - XCTAssertEqual(result.numberOfFailureOptOuts, 3) - XCTAssertEqual(result.numberOfNewMatchesFound, 3) - XCTAssertEqual(result.numberOfReAppereances, 3) + XCTAssertEqual(result.numberOfProfilesFound, 8) + XCTAssertEqual(result.numberOfOptOutsInProgress, 4) + XCTAssertEqual(result.numberOfSuccessfulOptOuts, 2) + XCTAssertEqual(result.numberOfFailureOptOuts, 2) + XCTAssertEqual(result.numberOfNewMatchesFound, 2) + XCTAssertEqual(result.numberOfReAppereances, 2) } /// This test data has the following parameters