Merge branch 'main' into diego/domain-exclusion-improvements

# By Daniel Bernal (4) and others # Via Bartek Waresiak (2) and others * main: (32 commits) Add error pixels for Subscription keychain access errors (#3276) [DuckPlayer] 24. FrontEnd Translation (#3281) [DuckPlayer] 23. Ship Review Fixes (#3279) Logging refactoring phase #2 (#3268) Move WireGuard dependency to packet tunnel provider (#3273) Add PPro unified feedback form (#3248) disable flaky test (#3283) Unified feedback form for Privacy Pro (#3172) Release 7.135.0-1 (#3280) fix back forward swipe gesture in landscape (#3278) Set onboarding completed for sync end to end tests (#3259) Release 7.135.0-0 (#3277) BSK Bump for macOS Freemium PIR RMF Attribute (No iOS Impact) (#3275) Fix math expressions on address bar (#3262) fix progress view crash by removing call to flush (#3272) Update autoconsent to v10.15.0 (#3254) Mitigate Onboarding pixels firing with empty atb (#3265) Clean up legacy VPN navigation (#3266) Re-Enable Broken Site Toast on iOS (#3244) Bump BSK (No iOS Changes) (#3261) ... # Conflicts: # DuckDuckGo.xcodeproj/project.pbxproj # DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
duckduckgo · Aug 29, 2024 · fb16ec6 · fb16ec6
2 parents 87f399a + aeb6f67
commit fb16ec6
Show file tree

Hide file tree

Showing 205 changed files with 21,079 additions and 7,282 deletions.
diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml
@@ -6,10 +6,11 @@ on:
   workflow_dispatch:
 
 jobs:
-  end-to-end-tests:
-    name: End to end Tests
+  build-end-to-end-tests:
+    name: Build End to End Tests
     runs-on: macos-14-xlarge
-
+    timeout-minutes: 30
+
     steps:
     - name: Check out the code
       uses: actions/checkout@v3
@@ -48,26 +49,59 @@ jobs:
           ONLY_ACTIVE_ARCH=NO \
         | tee xcodebuild.log
 
-    - name: Install Maestro
-      run: |
-        export MAESTRO_VERSION=1.36.0; curl -Ls "https://get.maestro.mobile.dev" | bash
+    - name: Store Binary
+      uses: actions/upload-artifact@v4
+      with:
+        name: duckduckgo-ios-app
+        path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app
+
+    - name: Upload logs when workflow failed
+      uses: actions/upload-artifact@v4
+      if: failure() || cancelled()
+      with:
+        name: BuildLogs
+        path: |
+          xcodebuild.log
+          DerivedData/Logs/Test/*.xcresult
+        retention-days: 7
 
-    - name: Release tests
-      run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=release DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+  end-to-end-tests:
+    name: End to end Tests
+    needs: build-end-to-end-tests
+    runs-on: macos-14-xlarge
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        test-tag: [release, privacy, securityTest, adClick]
+      max-parallel: 1 # Uncomment this line to run tests sequentially. 
+      fail-fast: false
 
-    - name: Privacy tests
-      run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=privacy DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+    steps:
+
+    - name: Check out the code
+      uses: actions/checkout@v3 # Don't need submodules here as this is only for the tests folder
+
+    - name: Retrieve Binary
+      uses: actions/download-artifact@v4
+      with:
+        name: duckduckgo-ios-app
+        path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app
 
-    - name: Security tests
+    - name: Install Maestro
       run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=securityTest DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+        export MAESTRO_VERSION=1.36.0; curl -Ls "https://get.maestro.mobile.dev" | bash
 
-    - name: Ad Click Detection Flow tests
+    - name: End to End tests
       run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=adClick DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=${{ matrix.test-tag }} DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
       
+  notify-failure:
+    name: Notify on failure
+    if: ${{ always() && contains(join(needs.*.result, ','), 'failure') && github.ref_name == 'main' }}
+    needs: [build-end-to-end-tests, end-to-end-tests]
+    runs-on: ubuntu-latest
+
+    steps:
     - name: Create Asana task when workflow failed
       if: ${{ failure() }}
       run: |
@@ -76,13 +110,3 @@ jobs:
           --header "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
           --header "Content-Type: application/json" \
           --data ' { "data": { "name": "GH Workflow Failure - End to end tests", "workspace": "${{ vars.GH_ASANA_WORKSPACE_ID }}", "projects": [ "${{ vars.GH_ASANA_IOS_APP_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/duckduckgo/iOS/actions/runs/${{ github.run_id }}. For instructions on how to handle the failure(s), check https://app.asana.com/0/0/1206423571874502/f" } }'
-
-    - name: Upload logs when workflow failed
-      uses: actions/upload-artifact@v4
-      if: failure() || cancelled()
-      with:
-        name: BuildLogs
-        path: |
-          xcodebuild.log
-          DerivedData/Logs/Test/*.xcresult
-        retention-days: 7
diff --git a/.github/workflows/sync-end-to-end.yml b/.github/workflows/sync-end-to-end.yml
@@ -102,8 +102,8 @@ jobs:
         
     - name: Sync e2e tests 
       run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} --env=CODE=${{ steps.sync-recovery-code.outputs.recovery-code }} --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=${{ matrix.os-version }} --include-tags=sync DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
-        
+        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --env=CODE=${{ steps.sync-recovery-code.outputs.recovery-code }} --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=${{ matrix.os-version }} --include-tags=sync DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+
     - name: Reset config
       run: |
         git checkout .maestro/config.yaml

diff --git a/.maestro/data_clearing_tests/02_duckduckgo_settings.yml b/.maestro/data_clearing_tests/02_duckduckgo_settings.yml
@@ -17,7 +17,7 @@ tags:
 - pressKey: Enter
 
 # Change settings
-- tapOn: "Safe search: moderate"
+- tapOn: "Safe search"
 - tapOn: "Off"
 
 # Fire Button - twice, just to be sure

diff --git a/.maestro/release_tests/password-autofill.yaml b/.maestro/release_tests/password-autofill.yaml
@@ -45,12 +45,8 @@ tags:
 - tapOn: "Close"
 
 # Validate standard form
-- runFlow:
-    file: ../shared/delay.yaml
 - assertVisible:
     id: "searchEntry"
-- runFlow:
-    file: ../shared/delay.yaml
 - tapOn:
     id: "searchEntry"
 - runFlow:
@@ -71,11 +67,7 @@ tags:
 # Validate multistep form
 - tapOn: 
     id: "searchEntry"
-- runFlow:
-    file: ../shared/delay.yaml
 - inputText: "https://privacy-test-pages.site/autofill/autoprompt/3-multistep-form.html"
-- runFlow:
-    file: ../shared/delay.yaml
 - pressKey: Enter
 - runFlow:
     file: ../shared/delay.yaml

diff --git a/Configuration/Version.xcconfig b/Configuration/Version.xcconfig
@@ -1 +1 @@
-MARKETING_VERSION = 7.133.0
+MARKETING_VERSION = 7.135.0
diff --git a/Core/AppPrivacyConfigurationDataProvider.swift b/Core/AppPrivacyConfigurationDataProvider.swift
@@ -23,8 +23,8 @@ import BrowserServicesKit
 final public class AppPrivacyConfigurationDataProvider: EmbeddedDataProvider {
 
     public struct Constants {
-        public static let embeddedDataETag = "\"c2739bfb56babbdfb6c1bfaf2446ab76\""
-        public static let embeddedDataSHA = "76782f36ca36d0eb14bd73944ab76f5fa718af4e099f5e6e00edcec5a7ee4a04"
+        public static let embeddedDataETag = "\"3310d00f227a02b53a16ed4af90afee5\""
+        public static let embeddedDataSHA = "7371e7fd3293e42638aa3bd3905a205e70827b2fd980848af108c82cedabf8be"
     }
 
     public var embeddedDataEtag: String {

diff --git a/Core/AppTrackerDataSetProvider.swift b/Core/AppTrackerDataSetProvider.swift
@@ -23,8 +23,8 @@ import BrowserServicesKit
 final public class AppTrackerDataSetProvider: EmbeddedDataProvider {
 
     public struct Constants {
-        public static let embeddedDataETag = "\"077374b1c16cef9527d8bf11f59374a8\""
-        public static let embeddedDataSHA = "6a9794936a6f569cc7c442a3b03802e220a7b194d6ab07e0459feb456fba92a1"
+        public static let embeddedDataETag = "\"4f9b48a36688eee92064578bc1aebe0a\""
+        public static let embeddedDataSHA = "ee51cab5fd4b82e6751a3d94680c89d4e248e0d456650b443f115998e1d3bec3"
     }
 
     public var embeddedDataEtag: String {

diff --git a/Core/BookmarksImporter.swift b/Core/BookmarksImporter.swift
@@ -22,6 +22,7 @@ import Common
 import Foundation
 import Persistence
 import SwiftSoup
+import os.log
 
 public enum BookmarksImportError: Error {
     case invalidHtmlNoDLTag
@@ -94,7 +95,7 @@ final public class BookmarksImporter {
 
         if isDocumentInSafariFormat(document) {
             guard let newDocument = try transformSafariDocument(document: document) else {
-                os_log("Safari format could not be handled", type: .debug)
+                Logger.bookmarks.debug("Safari format could not be handled")
                 throw BookmarksImportError.safariTransformFailure
             }
             try parse(documentElement: newDocument, importedBookmark: nil)
@@ -208,7 +209,7 @@ final public class BookmarksImporter {
         do {
             try await coreDataStorage.importBookmarks(bookmarks)
         } catch {
-            os_log("Failed to save imported bookmarks to core data %s", type: .debug, error.localizedDescription)
+            Logger.bookmarks.error("Failed to save imported bookmarks to core data: \(error.localizedDescription, privacy: .public)")
             throw BookmarksImportError.saveFailure
         }
     }

diff --git a/Core/ContentBlocking.swift b/Core/ContentBlocking.swift
@@ -69,8 +69,7 @@ public final class ContentBlocking {
         contentBlockingManager = ContentBlockerRulesManager(rulesSource: contentBlockerRulesSource,
                                                             exceptionsSource: exceptionsSource,
                                                             lastCompiledRulesStore: lastCompiledRulesStore,
-                                                            errorReporting: Self.debugEvents,
-                                                            log: .contentBlockingLog)
+                                                            errorReporting: Self.debugEvents)
 
         adClickAttributionRulesProvider = AdClickAttributionRulesProvider(config: adClickAttribution,
                                                                           compiledRulesSource: contentBlockingManager,
@@ -139,17 +138,15 @@ public final class ContentBlocking {
         AdClickAttributionDetection(feature: adClickAttribution,
                                     tld: tld,
                                     eventReporting: attributionEvents,
-                                    errorReporting: attributionDebugEvents,
-                                    log: .adAttributionLog)
+                                    errorReporting: attributionDebugEvents)
     }
 
     public func makeAdClickAttributionLogic(tld: TLD) -> AdClickAttributionLogic {
         AdClickAttributionLogic(featureConfig: adClickAttribution,
                                 rulesProvider: adClickAttributionRulesProvider,
                                 tld: tld,
                                 eventReporting: attributionEvents,
-                                errorReporting: attributionDebugEvents,
-                                log: .adAttributionLog)
+                                errorReporting: attributionDebugEvents)
     }
 
     private let attributionEvents = EventMapping<AdClickAttributionEvents> { event, _, parameters, _ in

diff --git a/Core/CookieStorage.swift b/Core/CookieStorage.swift
@@ -19,6 +19,7 @@
 
 import Common
 import Foundation
+import os.log
 
 /// Class for persisting cookies for fire proofed sites to work around a WKWebView / DataStore bug which does not let data get persisted until the webview has loaded.
 ///
@@ -56,7 +57,7 @@ public class CookieStorage {
                     })
 
                     if let cookie = HTTPCookie(properties: properties) {
-                        os_log("read cookie %s %s %s", log: .generalLog, type: .debug, cookie.domain, cookie.name, cookie.value)
+                        Logger.general.debug("read cookie \(cookie.domain) \(cookie.name) \(cookie.value)")
                         storedCookies.append(cookie)
                     }
                 }

diff --git a/Core/DailyPixelFiring.swift b/Core/DailyPixelFiring.swift
@@ -0,0 +1,37 @@
+//
+//  DailyPixelFiring.swift
+//  DuckDuckGo
+//
+//  Copyright © 2024 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import Foundation
+
+public protocol DailyPixelFiring {
+    static func fireDaily(_ pixel: Pixel.Event,
+                          withAdditionalParameters params: [String: String])
+
+    static func fireDaily(_ pixel: Pixel.Event)
+}
+
+extension DailyPixel: DailyPixelFiring {
+    public static func fireDaily(_ pixel: Pixel.Event, withAdditionalParameters params: [String: String]) {
+        fire(pixel: pixel, withAdditionalParameters: params)
+    }
+
+    public static func fireDaily(_ pixel: Pixel.Event) {
+        fire(pixel: pixel)
+    }
+}
diff --git a/Core/DateExtension.swift b/Core/DateExtension.swift
@@ -28,4 +28,8 @@ extension Date {
     public func isLessThan48HoursAgo() -> Bool {
         self > Date().addingTimeInterval(-48 * 60 * 60)
     }
+
+    public func isLessThan(daysAgo days: Int) -> Bool {
+        self > Date().addingTimeInterval(Double(-days) * 24 * 60 * 60)
+    }
 }
diff --git a/Core/DebugUserScript.swift b/Core/DebugUserScript.swift
@@ -20,6 +20,7 @@
 import Common
 import WebKit
 import UserScript
+import os.log
 
 public class DebugUserScript: NSObject, UserScript {
 
@@ -56,7 +57,7 @@ public class DebugUserScript: NSObject, UserScript {
     }
 
     private func handleLog(message: WKScriptMessage) {
-        os_log("%s", log: .generalLog, type: .debug, String(describing: message.body))
+        Logger.general.debug("\(String(describing: message.body))")
     }
 
     private func handleSignpost(message: WKScriptMessage) {

diff --git a/Core/DefaultVariantManager.swift b/Core/DefaultVariantManager.swift
@@ -21,6 +21,7 @@ import Common
 import Foundation
 import Speech
 import BrowserServicesKit
+import os.log
 
 extension FeatureName {
     // Define your feature e.g.:
@@ -127,17 +128,17 @@ public class DefaultVariantManager: VariantManager {
 
     public func assignVariantIfNeeded(_ newInstallCompletion: (VariantManager) -> Void) {
         guard !storage.hasInstallStatistics else {
-            os_log("no new variant needed for existing user", log: .generalLog, type: .debug)
+            Logger.general.debug("no new variant needed for existing user")
             return
         }
 
         if let variant = currentVariant {
-            os_log("already assigned variant: %s", log: .generalLog, type: .debug, String(describing: variant))
+            Logger.general.debug("already assigned variant: \(String(describing: variant))")
             return
         }
 
         guard let variant = selectVariant() else {
-            os_log("Failed to assign variant", log: .generalLog, type: .debug)
+            Logger.general.debug("Failed to assign variant")
 
             // it's possible this failed because there are none to assign, we should still let new install logic execute
             _ = newInstallCompletion(self)

diff --git a/Core/EtagStorage.swift b/Core/EtagStorage.swift
@@ -20,6 +20,7 @@
 import Common
 import Foundation
 import Configuration
+import os.log
 
 public protocol BlockerListETagStorage {
 
@@ -36,7 +37,7 @@ public struct UserDefaultsETagStorage: BlockerListETagStorage {
 
     public func loadEtag(for configuration: Configuration) -> String? {
         let etag = defaults?.string(forKey: configuration.storeKey)
-        os_log("stored etag for %s %s", log: .generalLog, type: .debug, configuration.storeKey, etag ?? "nil")
+        Logger.general.debug("Stored etag for \(configuration.storeKey) \(etag ?? "nil")")
         return etag
     }
 

diff --git a/Core/HistoryManager.swift b/Core/HistoryManager.swift
@@ -23,6 +23,7 @@ import BrowserServicesKit
 import History
 import Common
 import Persistence
+import os.log
 
 public protocol HistoryManaging {
 
@@ -152,7 +153,7 @@ public class HistoryDatabase {
 
     public static var defaultDBLocation: URL = {
         guard let url = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first else {
-            os_log("HistoryDatabase.make - OUT, failed to get application support directory")
+            Logger.general.fault("HistoryDatabase.make - OUT, failed to get application support directory")
             fatalError("Failed to get location")
         }
         return url
@@ -163,18 +164,18 @@ public class HistoryDatabase {
     }()
 
     public static func make(location: URL = defaultDBLocation, readOnly: Bool = false) -> CoreDataDatabase {
-        os_log("HistoryDatabase.make - IN - %s", location.absoluteString)
+        Logger.general.debug("HistoryDatabase.make - IN - \(location.absoluteString)")
         let bundle = History.bundle
         guard let model = CoreDataDatabase.loadModel(from: bundle, named: "BrowsingHistory") else {
-            os_log("HistoryDatabase.make - OUT, failed to loadModel")
+            Logger.general.debug("HistoryDatabase.make - OUT, failed to loadModel")
             fatalError("Failed to load model")
         }
 
         let db = CoreDataDatabase(name: "History",
                                   containerLocation: location,
                                   model: model,
                                   readOnly: readOnly)
-        os_log("HistoryDatabase.make - OUT")
+        Logger.general.debug("HistoryDatabase.make - OUT")
         return db
     }
 }
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		MARKETING_VERSION = 7.133.0
		MARKETING_VERSION = 7.135.0