From ba89ebe67602332028fcf3ac98e03451e464f6c9 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Mon, 19 Aug 2024 19:26:06 +0100 Subject: [PATCH 1/4] Replaced streaming with chunking/paging in internal `deleteTiles` --- .../impls/objectbox/backend/backend.dart | 1 + .../internal_workers/standard/worker.dart | 172 ++++++++++-------- 2 files changed, 94 insertions(+), 79 deletions(-) diff --git a/lib/src/backend/impls/objectbox/backend/backend.dart b/lib/src/backend/impls/objectbox/backend/backend.dart index fb4c1b98..23fed73f 100644 --- a/lib/src/backend/impls/objectbox/backend/backend.dart +++ b/lib/src/backend/impls/objectbox/backend/backend.dart @@ -5,6 +5,7 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; import 'dart:isolate'; +import 'dart:math'; import 'package:collection/collection.dart'; import 'package:flutter/material.dart'; diff --git a/lib/src/backend/impls/objectbox/backend/internal_workers/standard/worker.dart b/lib/src/backend/impls/objectbox/backend/internal_workers/standard/worker.dart index 37f9e17c..3a34fb46 100644 --- a/lib/src/backend/impls/objectbox/backend/internal_workers/standard/worker.dart +++ b/lib/src/backend/impls/objectbox/backend/internal_workers/standard/worker.dart @@ -81,50 +81,64 @@ Future _worker( /// Note that a transaction is used internally as necessary. /// /// Returns the number of orphaned (deleted) tiles. - Future deleteTiles({ + int deleteTiles({ required Query storesQuery, required Query tilesQuery, - }) async { + int? limitTiles, + }) { + // Loading all tiles at once will consume too much memory + const chunkSize = 200; // Alternative to streaming: chunking/paging + final stores = root.box(); final tiles = root.box(); bool hadTilesToUpdate = false; int rootDeltaSize = 0; final tilesToRemove = []; - //final tileRelationsToUpdate = >[]; final storesToUpdate = {}; - final queriedStores = storesQuery.property(ObjectBoxStore_.name).find(); - if (queriedStores.isEmpty) return 0; - - // For each store, remove it from the tile if requested - // For each store & if removed, update that store's stats - await for (final tile in tilesQuery.stream()) { - tile.stores.removeWhere((store) { - if (!queriedStores.contains(store.name)) return false; - - storesToUpdate[store.name] = (storesToUpdate[store.name] ?? store) - ..length -= 1 - ..size -= tile.bytes.lengthInBytes; - - return true; - }); - - if (tile.stores.isNotEmpty) { - tile.stores.applyToDb(mode: PutMode.update); - hadTilesToUpdate = true; - continue; - } - - rootDeltaSize -= tile.bytes.lengthInBytes; - tilesToRemove.add(tile.id); - } - - if (!hadTilesToUpdate && tilesToRemove.isEmpty) return 0; - root.runInTransaction( TxMode.write, () { + final queriedStores = storesQuery.property(ObjectBoxStore_.name).find(); + if (queriedStores.isEmpty) return 0; + + final tileCount = + min(limitTiles ?? double.infinity, tilesQuery.count()); + if (tileCount == 0) return 0; + + for (int offset = 0; offset < tileCount; offset += chunkSize) { + final tilesChunk = (tilesQuery + ..offset = offset + ..limit = chunkSize) + .find(); + + // For each store, remove it from the tile if requested + // For each store & if removed, update that store's stats + for (final tile in tilesChunk) { + tile.stores.removeWhere((store) { + if (!queriedStores.contains(store.name)) return false; + + storesToUpdate[store.name] = (storesToUpdate[store.name] ?? store) + ..length -= 1 + ..size -= tile.bytes.lengthInBytes; + + return true; + }); + + if (tile.stores.isNotEmpty) { + tile.stores.applyToDb(mode: PutMode.update); + hadTilesToUpdate = true; + continue; + } + + rootDeltaSize -= tile.bytes.lengthInBytes; + tilesToRemove.add(tile.id); + } + } + + if (!hadTilesToUpdate && tilesToRemove.isEmpty) return 0; + tilesToRemove.forEach(tiles.remove); updateRootStatistics( @@ -302,21 +316,21 @@ Future _worker( )) .build(); - deleteTiles(storesQuery: storeQuery, tilesQuery: tilesQuery).then((_) { - stores.put( - store - ..length = 0 - ..size = 0 - ..hits = 0 - ..misses = 0, - mode: PutMode.update, - ); + deleteTiles(storesQuery: storeQuery, tilesQuery: tilesQuery); - sendRes(id: cmd.id); + stores.put( + store + ..length = 0 + ..size = 0 + ..hits = 0 + ..misses = 0, + mode: PutMode.update, + ); - storeQuery.close(); - tilesQuery.close(); - }); + sendRes(id: cmd.id); + + storeQuery.close(); + tilesQuery.close(); case _CmdType.renameStore: final currentStoreName = cmd.args['currentStoreName']! as String; final newStoreName = cmd.args['newStoreName']! as String; @@ -352,14 +366,14 @@ Future _worker( )) .build(); - deleteTiles(storesQuery: storesQuery, tilesQuery: tilesQuery).then((_) { - storesQuery.remove(); + deleteTiles(storesQuery: storesQuery, tilesQuery: tilesQuery); - sendRes(id: cmd.id); + storesQuery.remove(); - storesQuery.close(); - tilesQuery.close(); - }); + sendRes(id: cmd.id); + + storesQuery.close(); + tilesQuery.close(); case _CmdType.tileExistsInStore: final storeName = cmd.args['storeName']! as String; final url = cmd.args['url']! as String; @@ -436,16 +450,16 @@ Future _worker( .query(ObjectBoxTile_.url.equals(url)) .build(); - deleteTiles(storesQuery: storesQuery, tilesQuery: tilesQuery) - .then((orphans) { - sendRes( - id: cmd.id, - data: {'wasOrphan': orphans == 1}, - ); + final orphans = + deleteTiles(storesQuery: storesQuery, tilesQuery: tilesQuery); - storesQuery.close(); - tilesQuery.close(); - }); + sendRes( + id: cmd.id, + data: {'wasOrphan': orphans == 1}, + ); + + storesQuery.close(); + tilesQuery.close(); case _CmdType.registerHitOrMiss: final storeName = cmd.args['storeName']! as String; final hit = cmd.args['hit']! as bool; @@ -501,18 +515,19 @@ Future _worker( storeQuery.close(); tilesQuery.close(); } else { - tilesQuery.limit = numToRemove; + final orphans = deleteTiles( + storesQuery: storeQuery, + tilesQuery: tilesQuery, + limitTiles: numToRemove, + ); - deleteTiles(storesQuery: storeQuery, tilesQuery: tilesQuery) - .then((orphans) { - sendRes( - id: cmd.id, - data: {'numOrphans': orphans}, - ); + sendRes( + id: cmd.id, + data: {'numOrphans': orphans}, + ); - storeQuery.close(); - tilesQuery.close(); - }); + storeQuery.close(); + tilesQuery.close(); } case _CmdType.removeTilesOlderThan: final storeName = cmd.args['storeName']! as String; @@ -531,17 +546,16 @@ Future _worker( )) .build(); - deleteTiles(storesQuery: storesQuery, tilesQuery: tilesQuery) - .then((orphans) { - sendRes( - id: cmd.id, - data: {'numOrphans': orphans}, - ); + final orphans = + deleteTiles(storesQuery: storesQuery, tilesQuery: tilesQuery); - storesQuery.close(); - tilesQuery.close(); - }); + sendRes( + id: cmd.id, + data: {'numOrphans': orphans}, + ); + storesQuery.close(); + tilesQuery.close(); case _CmdType.readMetadata: final storeName = cmd.args['storeName']! as String; @@ -1109,7 +1123,7 @@ Future _worker( )) .build(); - await deleteTiles( + deleteTiles( storesQuery: storesQuery, tilesQuery: tilesQuery, ); From 8f7ebe1ce19b40454b86c9dfc0aa45ad06c58d6e Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Mon, 19 Aug 2024 22:30:06 +0100 Subject: [PATCH 2/4] Copied Android example app build fixes from v10 Copied workflow improvements from v10 --- .github/workflows/main.yml | 45 ++++++++++--------- example/android/.gitignore | 2 +- example/android/app/build.gradle | 29 ++++-------- example/android/build.gradle | 16 +------ example/android/gradle.properties | 5 +-- .../gradle/wrapper/gradle-wrapper.properties | 2 +- example/android/settings.gradle | 4 +- 7 files changed, 40 insertions(+), 63 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e3ea1f2d..1a06f7f2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@master + uses: actions/checkout@v4 - name: Run Dart Package Analyser uses: axel-op/dart-package-analyzer@master id: analysis @@ -35,11 +35,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@master + uses: actions/checkout@v4 - name: Setup Flutter Environment - uses: subosito/flutter-action@main + uses: subosito/flutter-action@v2 with: channel: "beta" + cache: true - name: Get Package Dependencies run: flutter pub get - name: Get Example Dependencies @@ -56,11 +57,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@master + uses: actions/checkout@v4 - name: Setup Flutter Environment - uses: subosito/flutter-action@main + uses: subosito/flutter-action@v2 with: channel: "beta" + cache: true - name: Get Dependencies run: flutter pub get - name: Install ObjectBox Libs For Testing @@ -77,20 +79,22 @@ jobs: working-directory: ./example steps: - name: Checkout Repository - uses: actions/checkout@master - - name: Setup Java 17 Environment - uses: actions/setup-java@v3 + uses: actions/checkout@v4 + - name: Setup Java 21 Environment + uses: actions/setup-java@v4 with: distribution: "temurin" - java-version: "17" + java-version: "21" + cache: 'gradle' - name: Setup Flutter Environment - uses: subosito/flutter-action@main + uses: subosito/flutter-action@v2 with: channel: "beta" + cache: true - name: Build run: flutter build apk --obfuscate --split-debug-info=./symbols - name: Upload Artifact - uses: actions/upload-artifact@v3.1.2 + uses: actions/upload-artifact@v4 with: name: android-demo path: example/build/app/outputs/apk/release @@ -105,18 +109,19 @@ jobs: working-directory: ./example steps: - name: Checkout Repository - uses: actions/checkout@master + uses: actions/checkout@v4 - name: Setup Flutter Environment - uses: subosito/flutter-action@main + uses: subosito/flutter-action@v2 with: channel: "beta" + cache: true - name: Build run: flutter build windows --obfuscate --split-debug-info=./symbols - name: Create Installer run: iscc "windowsApplicationInstallerSetup.iss" working-directory: . - name: Upload Artifact - uses: actions/upload-artifact@v3.1.2 + uses: actions/upload-artifact@v4 with: name: windows-demo path: windowsTemp/WindowsApplication.exe @@ -131,9 +136,9 @@ jobs: working-directory: ./tile_server steps: - name: Checkout Repository - uses: actions/checkout@master + uses: actions/checkout@v4 - name: Setup Dart Environment - uses: dart-lang/setup-dart@v1.6.2 + uses: dart-lang/setup-dart@v1 - name: Get Dependencies run: dart pub get - name: Get Dart Dependencies @@ -143,7 +148,7 @@ jobs: - name: Compile run: dart compile exe bin/tile_server.dart - name: Upload Artifact - uses: actions/upload-artifact@v4.3.1 + uses: actions/upload-artifact@v4 with: name: windows-ts path: tile_server/bin/tile_server.exe @@ -158,9 +163,9 @@ jobs: working-directory: ./tile_server steps: - name: Checkout Repository - uses: actions/checkout@master + uses: actions/checkout@v4 - name: Setup Dart Environment - uses: dart-lang/setup-dart@v1.6.2 + uses: dart-lang/setup-dart@v1 - name: Get Dependencies run: dart pub get - name: Run Pre-Compile Generator @@ -168,7 +173,7 @@ jobs: - name: Compile run: dart compile exe bin/tile_server.dart - name: Upload Artifact - uses: actions/upload-artifact@v3.1.2 + uses: actions/upload-artifact@v4 with: name: linux-ts path: tile_server/bin/tile_server.exe diff --git a/example/android/.gitignore b/example/android/.gitignore index 6f568019..55afd919 100644 --- a/example/android/.gitignore +++ b/example/android/.gitignore @@ -7,7 +7,7 @@ gradle-wrapper.jar GeneratedPluginRegistrant.java # Remember to never publicly share your keystore. -# See https://flutter.dev/docs/deployment/android#reference-the-keystore-from-the-app +# See https://flutter.dev/to/reference-keystore key.properties **/*.keystore **/*.jks diff --git a/example/android/app/build.gradle b/example/android/app/build.gradle index 6408bdf1..8e644b93 100644 --- a/example/android/app/build.gradle +++ b/example/android/app/build.gradle @@ -5,40 +5,27 @@ plugins { id "dev.flutter.flutter-gradle-plugin" } -def localProperties = new Properties() -def localPropertiesFile = rootProject.file("local.properties") -if (localPropertiesFile.exists()) { - localPropertiesFile.withReader("UTF-8") { reader -> - localProperties.load(reader) - } -} - -def flutterVersionCode = localProperties.getProperty("flutter.versionCode") -if (flutterVersionCode == null) { - flutterVersionCode = "9" -} - -def flutterVersionName = localProperties.getProperty("flutter.versionName") -if (flutterVersionName == null) { - flutterVersionName = "9.0" -} - android { namespace = "dev.jaffaketchup.fmtc.demo" compileSdk = flutter.compileSdkVersion - ndkVersion = flutter.ndkVersion + // ndkVersion = flutter.ndkVersion + ndkVersion = "26.1.10909125" compileOptions { sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 } + kotlinOptions { + jvmTarget = JavaVersion.VERSION_1_8 + } + defaultConfig { applicationId = "dev.jaffaketchup.fmtc.demo" minSdk = flutter.minSdkVersion targetSdk = flutter.targetSdkVersion - versionCode = flutterVersionCode.toInteger() - versionName = flutterVersionName + versionCode = flutter.versionCode + versionName = flutter.versionName } buildTypes { diff --git a/example/android/build.gradle b/example/android/build.gradle index d9ed5779..dd592163 100644 --- a/example/android/build.gradle +++ b/example/android/build.gradle @@ -8,23 +8,11 @@ allprojects { rootProject.buildDir = "../build" subprojects { - afterEvaluate { project -> - if (project.hasProperty('android')) { - project.android { - if (namespace == null) { - namespace project.group - } - } - } - } - project.buildDir = "${rootProject.buildDir}/${project.name}" -} - -subprojects { project.evaluationDependsOn(":app") } tasks.register("clean", Delete) { delete rootProject.buildDir -} \ No newline at end of file +} + diff --git a/example/android/gradle.properties b/example/android/gradle.properties index 5f5d39d0..25971708 100644 --- a/example/android/gradle.properties +++ b/example/android/gradle.properties @@ -1,6 +1,3 @@ -org.gradle.jvmargs=-Xmx4G -XX:+HeapDumpOnOutOfMemoryError +org.gradle.jvmargs=-Xmx4G -XX:MaxMetaspaceSize=2G -XX:+HeapDumpOnOutOfMemoryError android.useAndroidX=true android.enableJetifier=true -android.defaults.buildfeatures.buildconfig=true -android.nonTransitiveRClass=false -android.nonFinalResIds=false diff --git a/example/android/gradle/wrapper/gradle-wrapper.properties b/example/android/gradle/wrapper/gradle-wrapper.properties index 8bc9958a..3c85cfe0 100644 --- a/example/android/gradle/wrapper/gradle-wrapper.properties +++ b/example/android/gradle/wrapper/gradle-wrapper.properties @@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip diff --git a/example/android/settings.gradle b/example/android/settings.gradle index 65da7568..d3bb611e 100644 --- a/example/android/settings.gradle +++ b/example/android/settings.gradle @@ -18,8 +18,8 @@ pluginManagement { plugins { id "dev.flutter.flutter-plugin-loader" version "1.0.0" - id "com.android.application" version '8.0.1' apply false - id "org.jetbrains.kotlin.android" version "1.7.10" apply false + id "com.android.application" version '8.5.2' apply false + id "org.jetbrains.kotlin.android" version "1.8.22" apply false } include ":app" From 96ef3b0fa58d392ce6d290928a79752d4f34cdb2 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Mon, 19 Aug 2024 22:45:41 +0100 Subject: [PATCH 3/4] Improved internal documentation about `chunkSize` > `tilesChunkSize` Removed unused dependencies from example app --- example/pubspec.yaml | 2 -- .../backend/internal_workers/standard/worker.dart | 12 ++++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/example/pubspec.yaml b/example/pubspec.yaml index c39d9911..09672add 100644 --- a/example/pubspec.yaml +++ b/example/pubspec.yaml @@ -12,7 +12,6 @@ environment: dependencies: auto_size_text: ^3.0.0 badges: ^3.1.2 - better_open_file: ^3.6.5 collection: ^1.18.0 dart_earcut: ^1.1.0 file_picker: ^8.0.3 @@ -32,7 +31,6 @@ dependencies: provider: ^6.1.2 stream_transform: ^2.1.0 validators: ^3.0.0 - version: ^3.0.2 dependency_overrides: flutter_map_tile_caching: diff --git a/lib/src/backend/impls/objectbox/backend/internal_workers/standard/worker.dart b/lib/src/backend/impls/objectbox/backend/internal_workers/standard/worker.dart index 3a34fb46..46801c11 100644 --- a/lib/src/backend/impls/objectbox/backend/internal_workers/standard/worker.dart +++ b/lib/src/backend/impls/objectbox/backend/internal_workers/standard/worker.dart @@ -86,8 +86,12 @@ Future _worker( required Query tilesQuery, int? limitTiles, }) { - // Loading all tiles at once will consume too much memory - const chunkSize = 200; // Alternative to streaming: chunking/paging + // This requires processing potentially many tiles, but this will use too + // much memory. + // Streaming appears to be unstable. Therefore, chunking/paging using + // `offset` & `limit` is used in these situations. It may be less efficient, + // but more stable. + const tilesChunkSize = 200; final stores = root.box(); final tiles = root.box(); @@ -107,10 +111,10 @@ Future _worker( min(limitTiles ?? double.infinity, tilesQuery.count()); if (tileCount == 0) return 0; - for (int offset = 0; offset < tileCount; offset += chunkSize) { + for (int offset = 0; offset < tileCount; offset += tilesChunkSize) { final tilesChunk = (tilesQuery ..offset = offset - ..limit = chunkSize) + ..limit = tilesChunkSize) .find(); // For each store, remove it from the tile if requested From 777210601ef7b67a35ae3e02942a869fdd283afd Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Mon, 19 Aug 2024 22:52:24 +0100 Subject: [PATCH 4/4] Updated versioning and CHANGELOG --- CHANGELOG.md | 5 +++++ example/pubspec.yaml | 2 +- pubspec.yaml | 2 +- windowsApplicationInstallerSetup.iss | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 616559f9..a8d97917 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,11 @@ Many thanks to my sponsors, no matter how much or how little they donated. Spons # Changelog +## [9.1.3] - 2024/08/19 + +* Fixed bug where any operation that attempted to delete tiles fatally crashed on some iOS devices + This appears to be an [ObjectBox issue](https://github.com/objectbox/objectbox-dart/issues/654) where streaming the results of a database query caused the crash. Instead, FMTC now uses a custom chunking system to avoid streaming and also avoid loading potentially many tiles into memory. + ## [9.1.2] - 2024/08/07 * Fixed compilation on web platforms: FMTC now internally overrides the `FMTCObjectBoxBackend` and becomes a no-op diff --git a/example/pubspec.yaml b/example/pubspec.yaml index 09672add..f9474cce 100644 --- a/example/pubspec.yaml +++ b/example/pubspec.yaml @@ -3,7 +3,7 @@ description: The example application for 'flutter_map_tile_caching', showcasing it's functionality and use-cases. publish_to: "none" -version: 9.1.2 +version: 9.1.3 environment: sdk: ">=3.3.0 <4.0.0" diff --git a/pubspec.yaml b/pubspec.yaml index ed5a36e3..e479501c 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,7 +1,7 @@ name: flutter_map_tile_caching description: Plugin for 'flutter_map' providing advanced caching functionality, with ability to download map regions for offline use. -version: 9.1.2 +version: 9.1.3 repository: https://github.com/JaffaKetchup/flutter_map_tile_caching issue_tracker: https://github.com/JaffaKetchup/flutter_map_tile_caching/issues diff --git a/windowsApplicationInstallerSetup.iss b/windowsApplicationInstallerSetup.iss index 6fa0737a..50d87cc2 100644 --- a/windowsApplicationInstallerSetup.iss +++ b/windowsApplicationInstallerSetup.iss @@ -1,7 +1,7 @@ ; Script generated by the Inno Setup Script Wizard #define MyAppName "FMTC Demo" -#define MyAppVersion "for 9.1.2" +#define MyAppVersion "for 9.1.3" #define MyAppPublisher "JaffaKetchup Development" #define MyAppURL "https://github.com/JaffaKetchup/flutter_map_tile_caching" #define MyAppSupportURL "https://github.com/JaffaKetchup/flutter_map_tile_caching/issues"