From 844985bec9020e5e8a69d9bb2dae1648978dd4d8 Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 12:21:10 -0300 Subject: [PATCH 01/28] feat(metadata cache): introduces the new class PE-3786 --- lib/utils/metadata_cache.dart | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 lib/utils/metadata_cache.dart diff --git a/lib/utils/metadata_cache.dart b/lib/utils/metadata_cache.dart new file mode 100644 index 0000000000..9b729aad55 --- /dev/null +++ b/lib/utils/metadata_cache.dart @@ -0,0 +1,43 @@ +import 'dart:typed_data'; + +import 'package:stash/stash_api.dart'; + +const defaultMaxEntries = 20000; +const defaultCacheName = 'metadata-cache'; + +class MetadataCache { + final Cache _cache; + + MetadataCache(this._cache); + + Future put(String key, Uint8List data) async { + return _cache.put(key, data); + } + + Future get(String key) async { + return _cache.get(key); + } + + Future remove(String key) async { + return _cache.remove(key); + } + + Future clear() async { + return _cache.clear(); + } + + Future> get keys async { + return _cache.keys; + } + + static Future> newCacheFromStore( + CacheStore store, { + int maxEntries = defaultMaxEntries, + }) { + return store.cache( + name: defaultCacheName, + maxEntries: maxEntries, + evictionPolicy: const LfuEvictionPolicy(), + ); + } +} From c2de494e15fae38568d59ea02401c016fab09218 Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 12:21:35 -0300 Subject: [PATCH 02/28] test(metadata cache): unit tests its behaviors PE-3786 --- test/utils/metadata_cache_test.dart | 121 ++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 test/utils/metadata_cache_test.dart diff --git a/test/utils/metadata_cache_test.dart b/test/utils/metadata_cache_test.dart new file mode 100644 index 0000000000..2f56ffc36e --- /dev/null +++ b/test/utils/metadata_cache_test.dart @@ -0,0 +1,121 @@ +import 'dart:typed_data'; + +import 'package:ardrive/utils/metadata_cache.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:stash/stash_api.dart'; +import 'package:stash_memory/stash_memory.dart'; +import 'package:stash_shared_preferences/stash_shared_preferences.dart'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + group('MetadataCache class', () { + test('can be constructed out of a Cache', () async { + final memoryCache = await newMockCache(); + + expect(memoryCache, isInstanceOf>()); + expect(MetadataCache(memoryCache), isInstanceOf()); + }); + + test('will accept up to 10 entries', () async { + final memoryCache = await newMockCache(); + final metadataCache = MetadataCache(memoryCache); + + final mockData = generateMockData(10); + + for (int i = 0; i < mockData.length; i++) { + await metadataCache.put(i.toString(), mockData[i]); + expect(await metadataCache.get(i.toString()), mockData[i]); + } + + final keys = await metadataCache.keys; + expect(keys, ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']); + }); + + test('eviction policy LFU', () async { + final memoryCache = await newMockCache(); + final metadataCache = MetadataCache(memoryCache); + + final mockData = generateMockData(10); + + for (int i = 0; i < mockData.length; i++) { + await metadataCache.put(i.toString(), mockData[i]); + + if (i != 0) { + // These becomes the most frequently used items + /// and zero becomes the least frequently used + await metadataCache.get(i.toString()); + await metadataCache.get(i.toString()); + } + } + + expect(await metadataCache.get('0'), isNotNull); + + await metadataCache.put('eleventh-item', Uint8List.fromList([1])); + + expect( + await metadataCache.get('0'), + null, + ); + + final keys = await metadataCache.keys; + expect( + keys, + ['1', '2', '3', '4', '5', '6', '7', '8', '9', 'eleventh-item'], + ); + }); + + group('with a stash_shared_preferences cache', () { + late MetadataCache metadataCache; + + setUpAll(() { + SharedPreferences.setMockInitialValues({}); + }); + + test('can be constructed', () async { + final store = await newSharedPreferencesCacheStore(); + final cache = await MetadataCache.newCacheFromStore( + store, + maxEntries: 1, + ); + metadataCache = MetadataCache(cache); + + expect(cache, isInstanceOf>()); + expect(metadataCache, isInstanceOf()); + }); + + test('can write and read data', () async { + final fibonacciSequence = [0, 1, 1, 2, 3, 5, 8, 13, 21]; + + await metadataCache.put( + 'fibonacci', + Uint8List.fromList(fibonacciSequence), + ); + + final storedData = await metadataCache.get('fibonacci'); + expect(storedData, Uint8List.fromList(fibonacciSequence)); + + final keys = await metadataCache.keys; + expect(keys, ['fibonacci']); + }); + }); + }); +} + +List generateMockData(int count) { + final List mockData = []; + + for (int i = 0; i < count; i++) { + mockData.add(Uint8List.fromList([i])); + } + + assert(mockData.length == count); + + return mockData; +} + +Future> newMockCache() async { + final cacheStore = await newMemoryCacheStore(); + return MetadataCache.newCacheFromStore(cacheStore, maxEntries: 10); +} From 0b75bbd781e29e67da6af35c7f0e154f5cc02058 Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 12:22:35 -0300 Subject: [PATCH 03/28] feat(pubspec): adds stash_shared_preferences as dependency PE-3786 --- pubspec.lock | 8 ++++++++ pubspec.yaml | 1 + 2 files changed, 9 insertions(+) diff --git a/pubspec.lock b/pubspec.lock index 902a775816..5900f20924 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -1709,6 +1709,14 @@ packages: url: "https://pub.dev" source: hosted version: "4.5.3" + stash_shared_preferences: + dependency: "direct main" + description: + name: stash_shared_preferences + sha256: be81ec8f69c0a67475b77509558f78385dd09ee16493bb985312aad0442d4d59 + url: "https://pub.dev" + source: hosted + version: "4.6.2" stream_channel: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index fa77103d7f..cf88d9b246 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -104,6 +104,7 @@ dependencies: logger: ^1.3.0 flutter_stripe: ^9.1.1 flutter_stripe_web: ^4.0.1 + stash_shared_preferences: ^4.6.2 dev_dependencies: flutter_test: From 71c8555ec57b2370b8a6f956506308125de06d0e Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 14:15:24 -0300 Subject: [PATCH 04/28] feat(arweave service): cache the metadata at sync time PE-3786 --- lib/services/arweave/arweave_service.dart | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/services/arweave/arweave_service.dart b/lib/services/arweave/arweave_service.dart index 3e04aa6dc0..f9cdcbb4b9 100644 --- a/lib/services/arweave/arweave_service.dart +++ b/lib/services/arweave/arweave_service.dart @@ -10,6 +10,8 @@ import 'package:ardrive/utils/extensions.dart'; import 'package:ardrive/utils/graphql_retry.dart'; import 'package:ardrive/utils/http_retry.dart'; import 'package:ardrive/utils/internet_checker.dart'; +import 'package:ardrive/utils/logger/logger.dart'; +import 'package:ardrive/utils/metadata_cache.dart'; import 'package:ardrive/utils/snapshots/snapshot_drive_history.dart'; import 'package:ardrive/utils/snapshots/snapshot_item.dart'; import 'package:ardrive_http/ardrive_http.dart'; @@ -21,6 +23,7 @@ import 'package:drift/drift.dart'; import 'package:http/http.dart'; import 'package:package_info_plus/package_info_plus.dart'; import 'package:retry/retry.dart'; +import 'package:stash_shared_preferences/stash_shared_preferences.dart'; import 'error/gateway_response_handler.dart'; @@ -255,6 +258,12 @@ class ArweaveService { ), ); + final sharedPreferencesCacheStore = await newSharedPreferencesCacheStore(); + final sharedPreferencesCache = await MetadataCache.newCacheFromStore( + sharedPreferencesCacheStore, + ); + final metadataCache = MetadataCache(sharedPreferencesCache); + final blockHistory = []; for (var i = 0; i < entityTxs.length; i++) { @@ -276,6 +285,8 @@ class ArweaveService { final entityResponse = responses[i]; final rawEntityData = entityResponse; + await metadataCache.put(transaction.id, rawEntityData); + Entity? entity; if (entityType == EntityType.drive) { entity = await DriveEntity.fromTransaction( @@ -318,6 +329,9 @@ class ArweaveService { } } + final cacheMetadataKeys = await metadataCache.keys; + logger.d('Added ${cacheMetadataKeys.length} items to metadata cache'); + // Sort the entities in each block by ascending commit time. for (final block in blockHistory) { block.entities.removeWhere((e) => e == null); From 71b693956a032ac3725682dc96d526ae2cb79f65 Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 14:24:01 -0300 Subject: [PATCH 05/28] feat(create snapshot cubit): read metadata from cache; fallback to arweave PE-3786 --- .../create_snapshot/create_snapshot_cubit.dart | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/blocs/create_snapshot/create_snapshot_cubit.dart b/lib/blocs/create_snapshot/create_snapshot_cubit.dart index 5ad72b96a2..5b40e1795e 100644 --- a/lib/blocs/create_snapshot/create_snapshot_cubit.dart +++ b/lib/blocs/create_snapshot/create_snapshot_cubit.dart @@ -12,6 +12,7 @@ import 'package:ardrive/services/arweave/arweave.dart'; import 'package:ardrive/services/pst/pst.dart'; import 'package:ardrive/utils/ar_cost_to_usd.dart'; import 'package:ardrive/utils/html/html_util.dart'; +import 'package:ardrive/utils/metadata_cache.dart'; import 'package:ardrive/utils/snapshots/height_range.dart'; import 'package:ardrive/utils/snapshots/range.dart'; import 'package:ardrive/utils/snapshots/snapshot_item_to_be_created.dart'; @@ -20,6 +21,7 @@ import 'package:arweave/utils.dart'; import 'package:equatable/equatable.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter_bloc/flutter_bloc.dart'; +import 'package:stash_shared_preferences/stash_shared_preferences.dart'; import 'package:uuid/uuid.dart'; part 'create_snapshot_state.dart'; @@ -366,11 +368,17 @@ class CreateSnapshotCubit extends Cubit { await _driveDao.driveById(driveId: _driveId).getSingleOrNull(); final isPrivate = drive != null && drive.privacy != DrivePrivacy.public; - // gather from arweave if not cached - final Uint8List entityJsonData = await _arweave.dataFromTxId( - txId, - null, // key is null because we don't re-encrypt the snapshot data + final sharedPreferencesCacheStore = await newSharedPreferencesCacheStore(); + final sharedPreferencesCache = await MetadataCache.newCacheFromStore( + sharedPreferencesCacheStore, ); + final metadataCache = MetadataCache(sharedPreferencesCache); + + final Uint8List entityJsonData = await metadataCache.get(txId) ?? + await _arweave.dataFromTxId( + txId, + null, // key is null because we don't re-encrypt the snapshot data + ); if (isPrivate) { final safeEntityDataFromArweave = Uint8List.fromList( From 603e7eb640dc2f2baed88173730c6087e6df4265 Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 14:30:26 -0300 Subject: [PATCH 06/28] feat(create snapshot cubit): cache the fetched metadata on snapshot creation time PE-3786 --- lib/blocs/create_snapshot/create_snapshot_cubit.dart | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/blocs/create_snapshot/create_snapshot_cubit.dart b/lib/blocs/create_snapshot/create_snapshot_cubit.dart index 5b40e1795e..89053f1b27 100644 --- a/lib/blocs/create_snapshot/create_snapshot_cubit.dart +++ b/lib/blocs/create_snapshot/create_snapshot_cubit.dart @@ -374,12 +374,19 @@ class CreateSnapshotCubit extends Cubit { ); final metadataCache = MetadataCache(sharedPreferencesCache); - final Uint8List entityJsonData = await metadataCache.get(txId) ?? + final Uint8List? cachedMetadata = await metadataCache.get(txId); + + final Uint8List entityJsonData = cachedMetadata ?? await _arweave.dataFromTxId( txId, null, // key is null because we don't re-encrypt the snapshot data ); + if (cachedMetadata == null) { + // Write to the cache the data we just fetched + await metadataCache.put(txId, entityJsonData); + } + if (isPrivate) { final safeEntityDataFromArweave = Uint8List.fromList( utf8.encode(base64Encode(entityJsonData)), From f63a54f2a4a566d8acbe57621e341d03ac3b876c Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 14:31:12 -0300 Subject: [PATCH 07/28] feat(metadata cache): adds log statements PE-3786 --- lib/utils/metadata_cache.dart | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/utils/metadata_cache.dart b/lib/utils/metadata_cache.dart index 9b729aad55..b06dc86a17 100644 --- a/lib/utils/metadata_cache.dart +++ b/lib/utils/metadata_cache.dart @@ -1,5 +1,6 @@ import 'dart:typed_data'; +import 'package:ardrive/utils/logger/logger.dart'; import 'package:stash/stash_api.dart'; const defaultMaxEntries = 20000; @@ -8,14 +9,21 @@ const defaultCacheName = 'metadata-cache'; class MetadataCache { final Cache _cache; - MetadataCache(this._cache); + const MetadataCache(this._cache); Future put(String key, Uint8List data) async { + logger.d('Putting $key in cache'); return _cache.put(key, data); } Future get(String key) async { - return _cache.get(key); + final value = await _cache.get(key); + if (value != null) { + logger.d('Cache hit for $key'); + } else { + logger.d('Cache miss for $key'); + } + return value; } Future remove(String key) async { From df040f99731e1dd998e0ae3d4c6b5a402a01929c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Batista?= Date: Fri, 19 May 2023 17:00:48 -0300 Subject: [PATCH 08/28] chore(arweave service): removes unnecessary log statement PE-3786 --- lib/services/arweave/arweave_service.dart | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/services/arweave/arweave_service.dart b/lib/services/arweave/arweave_service.dart index f9cdcbb4b9..42e0bca021 100644 --- a/lib/services/arweave/arweave_service.dart +++ b/lib/services/arweave/arweave_service.dart @@ -329,9 +329,6 @@ class ArweaveService { } } - final cacheMetadataKeys = await metadataCache.keys; - logger.d('Added ${cacheMetadataKeys.length} items to metadata cache'); - // Sort the entities in each block by ascending commit time. for (final block in blockHistory) { block.entities.removeWhere((e) => e == null); From bc50ee2facbe2fbc0665f93976e86950fa3ea656 Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 17:07:48 -0300 Subject: [PATCH 09/28] feat(arweave service): replace print statements with logger PE-3786 --- lib/services/arweave/arweave_service.dart | 29 +++++++++++++---------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/lib/services/arweave/arweave_service.dart b/lib/services/arweave/arweave_service.dart index 42e0bca021..b0ed08fb1e 100644 --- a/lib/services/arweave/arweave_service.dart +++ b/lib/services/arweave/arweave_service.dart @@ -54,7 +54,7 @@ class ArweaveService { GatewayResponseHandler(), HttpRetryOptions(onRetry: (exception) { if (exception is GatewayError) { - print( + logger.i( 'Retrying for ${exception.runtimeType} exception\n' 'for route ${exception.requestUrl}\n' 'and status code ${exception.statusCode}', @@ -62,7 +62,7 @@ class ArweaveService { return; } - print('Retrying for unknown exception: ${exception.toString()}'); + logger.w('Retrying for unknown exception: ${exception.toString()}'); }, retryIf: (exception) { return exception is! RateLimitError; })); @@ -245,7 +245,7 @@ class ArweaveService { // don't fetch data for snapshots if (isSnapshot) { - print('skipping unnecessary request for snapshot data'); + logger.d('skipping unnecessary request for snapshot data'); return Uint8List(0); } @@ -315,12 +315,12 @@ class ArweaveService { // If there are errors in parsing the entity, ignore it. } on EntityTransactionParseException catch (parseException) { - print( + logger.w( 'Failed to parse transaction ' 'with id ${parseException.transactionId}', ); } on GatewayError catch (fetchException) { - print( + logger.w( 'Failed to fetch entity data with the exception ${fetchException.runtimeType}' 'for transaction ${transaction.id}, ' 'with status ${fetchException.statusCode} ' @@ -472,8 +472,9 @@ class ArweaveService { () async => await Future.wait( driveTxs.map((e) => client.api.getSandboxedTx(e.id)), ), onRetry: (Exception err) { - print( - 'Retrying for get unique user drive entities on Exception: ${err.toString()}'); + logger.i( + 'Retrying for get unique user drive entities on Exception: ${err.toString()}', + ); }); final drivesById = {}; @@ -508,7 +509,7 @@ class ArweaveService { // If there's an error parsing the drive entity, just ignore it. } on EntityTransactionParseException catch (parseException) { - print( + logger.w( 'Failed to parse transaction ' 'with id ${parseException.transactionId}', ); @@ -516,8 +517,10 @@ class ArweaveService { } return drivesWithKey; } catch (e, stacktrace) { - print( - 'An error occurs getting the unique user drive entities. Exception: ${e.toString()} stacktrace: ${stacktrace.toString()}'); + logger.w( + 'An error occurred when getting the unique user drive entities.' + ' Exception: ${e.toString()} stacktrace: ${stacktrace.toString()}', + ); rethrow; } } @@ -568,7 +571,7 @@ class ArweaveService { return await DriveEntity.fromTransaction( fileTx, _crypto, fileDataRes.bodyBytes, driveKey); } on EntityTransactionParseException catch (parseException) { - print( + logger.w( 'Failed to parse transaction ' 'with id ${parseException.transactionId}', ); @@ -730,7 +733,7 @@ class ArweaveService { crypto: _crypto, ); } on EntityTransactionParseException catch (parseException) { - print( + logger.w( 'Failed to parse transaction ' 'with id ${parseException.transactionId}', ); @@ -847,7 +850,7 @@ class ArweaveService { try { await Future.wait(confirmationFutures); } catch (e) { - print('Error getting transactions confirmations on exception: $e'); + logger.w('Error getting transactions confirmations on exception: $e'); rethrow; } From f05e7d8f869b2288b59ce44f3ac3f2dba65ccce7 Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 17:12:49 -0300 Subject: [PATCH 10/28] test(metadata cache): test the remove and clear methods PE-3786 --- test/utils/metadata_cache_test.dart | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/utils/metadata_cache_test.dart b/test/utils/metadata_cache_test.dart index 2f56ffc36e..7ba4ebcce5 100644 --- a/test/utils/metadata_cache_test.dart +++ b/test/utils/metadata_cache_test.dart @@ -66,6 +66,39 @@ void main() { ); }); + test('can remove added items', () async { + final memoryCache = await newMockCache(); + final metadataCache = MetadataCache(memoryCache); + + final mockData = generateMockData(10); + + for (int i = 0; i < mockData.length; i++) { + await metadataCache.put(i.toString(), mockData[i]); + expect(await metadataCache.get(i.toString()), mockData[i]); + await metadataCache.remove(i.toString()); + expect(await metadataCache.get(i.toString()), null); + } + }); + + test('can be cleared', () async { + final memoryCache = await newMockCache(); + final metadataCache = MetadataCache(memoryCache); + + final mockData = generateMockData(10); + + for (int i = 0; i < mockData.length; i++) { + await metadataCache.put(i.toString(), mockData[i]); + } + + final keys = await metadataCache.keys; + expect(keys, ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']); + + await metadataCache.clear(); + + final keysAfterClear = await metadataCache.keys; + expect(keysAfterClear, []); + }); + group('with a stash_shared_preferences cache', () { late MetadataCache metadataCache; From 08486f9f48307f675b7c8b185a560ed55630b4e1 Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 17:16:34 -0300 Subject: [PATCH 11/28] feat(metadata cache): improves logging PE-3786 --- lib/utils/metadata_cache.dart | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/utils/metadata_cache.dart b/lib/utils/metadata_cache.dart index b06dc86a17..a181f38cab 100644 --- a/lib/utils/metadata_cache.dart +++ b/lib/utils/metadata_cache.dart @@ -12,25 +12,27 @@ class MetadataCache { const MetadataCache(this._cache); Future put(String key, Uint8List data) async { - logger.d('Putting $key in cache'); + logger.d('Putting $key in metadata cache'); return _cache.put(key, data); } Future get(String key) async { final value = await _cache.get(key); if (value != null) { - logger.d('Cache hit for $key'); + logger.d('Cache hit for $key in metadata cache'); } else { - logger.d('Cache miss for $key'); + logger.d('Cache miss for $key in metadata cache'); } return value; } Future remove(String key) async { + logger.d('Removing $key from metadata cache'); return _cache.remove(key); } Future clear() async { + logger.d('Clearing metadata cache'); return _cache.clear(); } From 3a706784ad9a90085dcddf10dfc42f48faab02cc Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 17:39:43 -0300 Subject: [PATCH 12/28] feat(metadata cache): simplifies the constructor PE-3786 --- .../create_snapshot_cubit.dart | 6 +-- lib/services/arweave/arweave_service.dart | 6 +-- lib/utils/metadata_cache.dart | 10 ++++- test/utils/metadata_cache_test.dart | 39 +++++++++---------- 4 files changed, 32 insertions(+), 29 deletions(-) diff --git a/lib/blocs/create_snapshot/create_snapshot_cubit.dart b/lib/blocs/create_snapshot/create_snapshot_cubit.dart index 89053f1b27..4f2f8376c3 100644 --- a/lib/blocs/create_snapshot/create_snapshot_cubit.dart +++ b/lib/blocs/create_snapshot/create_snapshot_cubit.dart @@ -368,11 +368,9 @@ class CreateSnapshotCubit extends Cubit { await _driveDao.driveById(driveId: _driveId).getSingleOrNull(); final isPrivate = drive != null && drive.privacy != DrivePrivacy.public; - final sharedPreferencesCacheStore = await newSharedPreferencesCacheStore(); - final sharedPreferencesCache = await MetadataCache.newCacheFromStore( - sharedPreferencesCacheStore, + final metadataCache = await MetadataCache.fromCacheStore( + await newSharedPreferencesCacheStore(), ); - final metadataCache = MetadataCache(sharedPreferencesCache); final Uint8List? cachedMetadata = await metadataCache.get(txId); diff --git a/lib/services/arweave/arweave_service.dart b/lib/services/arweave/arweave_service.dart index b0ed08fb1e..fcf171268a 100644 --- a/lib/services/arweave/arweave_service.dart +++ b/lib/services/arweave/arweave_service.dart @@ -258,11 +258,9 @@ class ArweaveService { ), ); - final sharedPreferencesCacheStore = await newSharedPreferencesCacheStore(); - final sharedPreferencesCache = await MetadataCache.newCacheFromStore( - sharedPreferencesCacheStore, + final metadataCache = await MetadataCache.fromCacheStore( + await newSharedPreferencesCacheStore(), ); - final metadataCache = MetadataCache(sharedPreferencesCache); final blockHistory = []; diff --git a/lib/utils/metadata_cache.dart b/lib/utils/metadata_cache.dart index a181f38cab..7cd980fd48 100644 --- a/lib/utils/metadata_cache.dart +++ b/lib/utils/metadata_cache.dart @@ -11,6 +11,14 @@ class MetadataCache { const MetadataCache(this._cache); + static Future fromCacheStore( + CacheStore store, { + int maxEntries = defaultMaxEntries, + }) async { + final cache = await _newCacheFromStore(store, maxEntries: maxEntries); + return MetadataCache(cache); + } + Future put(String key, Uint8List data) async { logger.d('Putting $key in metadata cache'); return _cache.put(key, data); @@ -40,7 +48,7 @@ class MetadataCache { return _cache.keys; } - static Future> newCacheFromStore( + static Future> _newCacheFromStore( CacheStore store, { int maxEntries = defaultMaxEntries, }) { diff --git a/test/utils/metadata_cache_test.dart b/test/utils/metadata_cache_test.dart index 7ba4ebcce5..66905fe847 100644 --- a/test/utils/metadata_cache_test.dart +++ b/test/utils/metadata_cache_test.dart @@ -11,16 +11,19 @@ void main() { TestWidgetsFlutterBinding.ensureInitialized(); group('MetadataCache class', () { - test('can be constructed out of a Cache', () async { - final memoryCache = await newMockCache(); + test('can be constructed out of a Cache Store', () async { + final CacheStore cacheStore = await newMemoryCacheStore(); - expect(memoryCache, isInstanceOf>()); - expect(MetadataCache(memoryCache), isInstanceOf()); + expect( + await MetadataCache.fromCacheStore(cacheStore), + isInstanceOf(), + ); }); test('will accept up to 10 entries', () async { - final memoryCache = await newMockCache(); - final metadataCache = MetadataCache(memoryCache); + final metadataCache = await MetadataCache.fromCacheStore( + await newMemoryCacheStore(), + ); final mockData = generateMockData(10); @@ -34,8 +37,9 @@ void main() { }); test('eviction policy LFU', () async { - final memoryCache = await newMockCache(); - final metadataCache = MetadataCache(memoryCache); + final metadataCache = await MetadataCache.fromCacheStore( + await newMemoryCacheStore(), + ); final mockData = generateMockData(10); @@ -67,8 +71,9 @@ void main() { }); test('can remove added items', () async { - final memoryCache = await newMockCache(); - final metadataCache = MetadataCache(memoryCache); + final metadataCache = await MetadataCache.fromCacheStore( + await newMemoryCacheStore(), + ); final mockData = generateMockData(10); @@ -81,8 +86,9 @@ void main() { }); test('can be cleared', () async { - final memoryCache = await newMockCache(); - final metadataCache = MetadataCache(memoryCache); + final metadataCache = await MetadataCache.fromCacheStore( + await newMemoryCacheStore(), + ); final mockData = generateMockData(10); @@ -108,13 +114,11 @@ void main() { test('can be constructed', () async { final store = await newSharedPreferencesCacheStore(); - final cache = await MetadataCache.newCacheFromStore( + metadataCache = await MetadataCache.fromCacheStore( store, maxEntries: 1, ); - metadataCache = MetadataCache(cache); - expect(cache, isInstanceOf>()); expect(metadataCache, isInstanceOf()); }); @@ -147,8 +151,3 @@ List generateMockData(int count) { return mockData; } - -Future> newMockCache() async { - final cacheStore = await newMemoryCacheStore(); - return MetadataCache.newCacheFromStore(cacheStore, maxEntries: 10); -} From 0b0a72c837f25ee387bf308b162fc0948c521712 Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 17:41:01 -0300 Subject: [PATCH 13/28] feat(ardrive auth): clear metadata cache on logout PE-3786 --- lib/authentication/ardrive_auth.dart | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/authentication/ardrive_auth.dart b/lib/authentication/ardrive_auth.dart index dc38934ef8..dc50eb0cb5 100644 --- a/lib/authentication/ardrive_auth.dart +++ b/lib/authentication/ardrive_auth.dart @@ -9,9 +9,11 @@ import 'package:ardrive/user/repositories/user_repository.dart'; import 'package:ardrive/user/user.dart'; import 'package:ardrive/utils/constants.dart'; import 'package:ardrive/utils/logger/logger.dart'; +import 'package:ardrive/utils/metadata_cache.dart'; import 'package:ardrive/utils/secure_key_value_store.dart'; import 'package:arweave/arweave.dart'; import 'package:cryptography/cryptography.dart'; +import 'package:stash_shared_preferences/stash_shared_preferences.dart'; import '../core/crypto/crypto.dart'; @@ -185,6 +187,12 @@ class _ArDriveAuth implements ArDriveAuth { } await _databaseHelpers.deleteAllTables(); + + final metadataCache = await MetadataCache.fromCacheStore( + await newSharedPreferencesCacheStore(), + ); + + metadataCache.clear(); } catch (e) { logger.e('Failed to logout user', e); throw AuthenticationFailedException('Failed to logout user'); From 1e53c672a2079be2e846ba8d39a9664c846d83e0 Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 17:54:00 -0300 Subject: [PATCH 14/28] test(metadata cache): updates broken tests PE-3786 --- test/utils/metadata_cache_test.dart | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/utils/metadata_cache_test.dart b/test/utils/metadata_cache_test.dart index 66905fe847..6ac114d3a4 100644 --- a/test/utils/metadata_cache_test.dart +++ b/test/utils/metadata_cache_test.dart @@ -23,6 +23,7 @@ void main() { test('will accept up to 10 entries', () async { final metadataCache = await MetadataCache.fromCacheStore( await newMemoryCacheStore(), + maxEntries: 10, ); final mockData = generateMockData(10); @@ -39,6 +40,7 @@ void main() { test('eviction policy LFU', () async { final metadataCache = await MetadataCache.fromCacheStore( await newMemoryCacheStore(), + maxEntries: 10, ); final mockData = generateMockData(10); @@ -73,6 +75,7 @@ void main() { test('can remove added items', () async { final metadataCache = await MetadataCache.fromCacheStore( await newMemoryCacheStore(), + maxEntries: 10, ); final mockData = generateMockData(10); @@ -88,6 +91,7 @@ void main() { test('can be cleared', () async { final metadataCache = await MetadataCache.fromCacheStore( await newMemoryCacheStore(), + maxEntries: 10, ); final mockData = generateMockData(10); From 0b709bb53f243a20fc4a3a4560b3f7b359cfdcc5 Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 18:57:16 -0300 Subject: [PATCH 15/28] feat(ardrive auth): clear metadata cache on logout PE-3786 --- lib/authentication/ardrive_auth.dart | 20 +++++++++++++------ .../login/blocs/login_bloc.dart | 4 +++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/authentication/ardrive_auth.dart b/lib/authentication/ardrive_auth.dart index dc50eb0cb5..0f0cc7fb09 100644 --- a/lib/authentication/ardrive_auth.dart +++ b/lib/authentication/ardrive_auth.dart @@ -36,6 +36,7 @@ abstract class ArDriveAuth { required SecureKeyValueStore secureKeyValueStore, required ArConnectService arConnectService, required DatabaseHelpers databaseHelpers, + MetadataCache? metadataCache, }) => _ArDriveAuth( arweave: arweave, @@ -45,6 +46,7 @@ abstract class ArDriveAuth { biometricAuthentication: biometricAuthentication, secureKeyValueStore: secureKeyValueStore, arConnectService: arConnectService, + metadataCache: metadataCache, ); } @@ -57,13 +59,15 @@ class _ArDriveAuth implements ArDriveAuth { required SecureKeyValueStore secureKeyValueStore, required ArConnectService arConnectService, required DatabaseHelpers databaseHelpers, + MetadataCache? metadataCache, }) : _arweave = arweave, _crypto = crypto, _databaseHelpers = databaseHelpers, _arConnectService = arConnectService, _secureKeyValueStore = secureKeyValueStore, _biometricAuthentication = biometricAuthentication, - _userRepository = userRepository; + _userRepository = userRepository, + _maybeMetadataCache = metadataCache; final UserRepository _userRepository; final ArweaveService _arweave; @@ -72,6 +76,7 @@ class _ArDriveAuth implements ArDriveAuth { final SecureKeyValueStore _secureKeyValueStore; final ArConnectService _arConnectService; final DatabaseHelpers _databaseHelpers; + MetadataCache? _maybeMetadataCache; User? _currentUser; @@ -89,6 +94,13 @@ class _ArDriveAuth implements ArDriveAuth { _currentUser = user; } + Future get metadataCache async { + _maybeMetadataCache ??= await MetadataCache.fromCacheStore( + await newSharedPreferencesCacheStore(), + ); + return _maybeMetadataCache!; + } + final StreamController _userStreamController = StreamController.broadcast(); @@ -188,11 +200,7 @@ class _ArDriveAuth implements ArDriveAuth { await _databaseHelpers.deleteAllTables(); - final metadataCache = await MetadataCache.fromCacheStore( - await newSharedPreferencesCacheStore(), - ); - - metadataCache.clear(); + (await metadataCache).clear(); } catch (e) { logger.e('Failed to logout user', e); throw AuthenticationFailedException('Failed to logout user'); diff --git a/lib/authentication/login/blocs/login_bloc.dart b/lib/authentication/login/blocs/login_bloc.dart index b63cf950c8..8c3fd63984 100644 --- a/lib/authentication/login/blocs/login_bloc.dart +++ b/lib/authentication/login/blocs/login_bloc.dart @@ -224,7 +224,9 @@ class LoginBloc extends Bloc { } Future _handleForgetWalletEvent( - ForgetWallet event, Emitter emit) async { + ForgetWallet event, + Emitter emit, + ) async { if (await _arDriveAuth.isUserLoggedIn()) { await _arDriveAuth.logout(); } From 22938ada26473c0c8062b28d60a29e237f73e2ff Mon Sep 17 00:00:00 2001 From: Mati Date: Fri, 19 May 2023 18:57:51 -0300 Subject: [PATCH 16/28] test(ardrive auth): updates tests after refactor PE-3786 --- test/authentication/ardrive_auth_test.dart | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/authentication/ardrive_auth_test.dart b/test/authentication/ardrive_auth_test.dart index 9eb0e367c4..dd379ed85a 100644 --- a/test/authentication/ardrive_auth_test.dart +++ b/test/authentication/ardrive_auth_test.dart @@ -4,9 +4,11 @@ import 'package:ardrive/entities/profile_types.dart'; import 'package:ardrive/services/arweave/arweave.dart'; import 'package:ardrive/services/arweave/graphql/graphql_api.graphql.dart'; import 'package:ardrive/user/user.dart'; +import 'package:ardrive/utils/metadata_cache.dart'; import 'package:cryptography/cryptography.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:mocktail/mocktail.dart'; +import 'package:stash_memory/stash_memory.dart'; import '../test_utils/utils.dart'; @@ -25,7 +27,7 @@ void main() { final wallet = getTestWallet(); - setUp(() { + setUp(() async { mockArweaveService = MockArweaveService(); mockUserRepository = MockUserRepository(); mockArDriveCrypto = MockArDriveCrypto(); @@ -34,6 +36,10 @@ void main() { mockArConnectService = MockArConnectService(); mockDatabaseHelpers = MockDatabaseHelpers(); + final metadataCache = await MetadataCache.fromCacheStore( + await newMemoryCacheStore(), + ); + arDriveAuth = ArDriveAuth( arweave: mockArweaveService, userRepository: mockUserRepository, @@ -42,6 +48,7 @@ void main() { arConnectService: mockArConnectService, biometricAuthentication: mockBiometricAuthentication, secureKeyValueStore: mockSecureKeyValueStore, + metadataCache: metadataCache, ); // register call back for test drive entity registerFallbackValue(DriveEntity( From dc42872cffbf1af727a513181cef2b9424944ac2 Mon Sep 17 00:00:00 2001 From: Mati Date: Mon, 22 May 2023 12:18:43 -0300 Subject: [PATCH 17/28] test(metadata cache): renames test case PE-3786 --- test/utils/metadata_cache_test.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/utils/metadata_cache_test.dart b/test/utils/metadata_cache_test.dart index 6ac114d3a4..84fd20b68e 100644 --- a/test/utils/metadata_cache_test.dart +++ b/test/utils/metadata_cache_test.dart @@ -20,7 +20,7 @@ void main() { ); }); - test('will accept up to 10 entries', () async { + test('can take as many entries as the maxEntries limit', () async { final metadataCache = await MetadataCache.fromCacheStore( await newMemoryCacheStore(), maxEntries: 10, From 80f4252c83a1111b605f5be6bf19d73673de3073 Mon Sep 17 00:00:00 2001 From: Mati Date: Mon, 22 May 2023 12:19:38 -0300 Subject: [PATCH 18/28] chore(metadata cache): puts a comment as reference to the Doc PE-3786 --- lib/utils/metadata_cache.dart | 2 ++ test/utils/metadata_cache_test.dart | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/utils/metadata_cache.dart b/lib/utils/metadata_cache.dart index 7cd980fd48..c55112dbe6 100644 --- a/lib/utils/metadata_cache.dart +++ b/lib/utils/metadata_cache.dart @@ -55,6 +55,8 @@ class MetadataCache { return store.cache( name: defaultCacheName, maxEntries: maxEntries, + + // See: https://pub.dev/packages/stash#eviction-policies evictionPolicy: const LfuEvictionPolicy(), ); } diff --git a/test/utils/metadata_cache_test.dart b/test/utils/metadata_cache_test.dart index 84fd20b68e..6c5b35a765 100644 --- a/test/utils/metadata_cache_test.dart +++ b/test/utils/metadata_cache_test.dart @@ -37,7 +37,14 @@ void main() { expect(keys, ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']); }); - test('eviction policy LFU', () async { + test('follows the eviction policy: Least Frequently Used (LFU)', () async { + // Refer to https://pub.dev/packages/stash#eviction-policies for more info + /// LfuEvictionPolicy LFU (least-frequently used) policy counts how often + /// an entry is used. Those that are least often used are discarded first. + /// In that sense it works very similarly to LRU except that instead of + /// storing the value of how recently a block was accessed, it stores the + /// value of how many times it was accessed + final metadataCache = await MetadataCache.fromCacheStore( await newMemoryCacheStore(), maxEntries: 10, From 1e0d7dbf8617b7b4e3a136fac4b6602cf0c318af Mon Sep 17 00:00:00 2001 From: Mati Date: Mon, 22 May 2023 12:20:31 -0300 Subject: [PATCH 19/28] test(metadata cache): remoes unnecessary assert PE-3786 --- test/utils/metadata_cache_test.dart | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/utils/metadata_cache_test.dart b/test/utils/metadata_cache_test.dart index 6c5b35a765..c2a436e31a 100644 --- a/test/utils/metadata_cache_test.dart +++ b/test/utils/metadata_cache_test.dart @@ -158,7 +158,5 @@ List generateMockData(int count) { mockData.add(Uint8List.fromList([i])); } - assert(mockData.length == count); - return mockData; } From 1b16e8092aaeeed767451443a43e42c9f4d69c49 Mon Sep 17 00:00:00 2001 From: Mati Date: Mon, 22 May 2023 12:25:37 -0300 Subject: [PATCH 20/28] chore(ardrive auth): makes the metadataCache property to be private PE-3786 --- lib/authentication/ardrive_auth.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/authentication/ardrive_auth.dart b/lib/authentication/ardrive_auth.dart index 0f0cc7fb09..abcfc0640d 100644 --- a/lib/authentication/ardrive_auth.dart +++ b/lib/authentication/ardrive_auth.dart @@ -94,7 +94,7 @@ class _ArDriveAuth implements ArDriveAuth { _currentUser = user; } - Future get metadataCache async { + Future get _metadataCache async { _maybeMetadataCache ??= await MetadataCache.fromCacheStore( await newSharedPreferencesCacheStore(), ); @@ -200,7 +200,7 @@ class _ArDriveAuth implements ArDriveAuth { await _databaseHelpers.deleteAllTables(); - (await metadataCache).clear(); + (await _metadataCache).clear(); } catch (e) { logger.e('Failed to logout user', e); throw AuthenticationFailedException('Failed to logout user'); From d2d6f4240f8734d1a77f27f35004f2bb40e6c3eb Mon Sep 17 00:00:00 2001 From: Mati Date: Thu, 25 May 2023 12:52:52 -0300 Subject: [PATCH 21/28] feat(metadata cache): implements custom no-eviction PE-3786 --- lib/utils/metadata_cache.dart | 36 ++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/lib/utils/metadata_cache.dart b/lib/utils/metadata_cache.dart index c55112dbe6..ad99458cbf 100644 --- a/lib/utils/metadata_cache.dart +++ b/lib/utils/metadata_cache.dart @@ -8,20 +8,29 @@ const defaultCacheName = 'metadata-cache'; class MetadataCache { final Cache _cache; + final int _maxEntries; - const MetadataCache(this._cache); + const MetadataCache(this._cache, {int maxEntries = defaultMaxEntries}) + : _maxEntries = maxEntries; static Future fromCacheStore( CacheStore store, { int maxEntries = defaultMaxEntries, }) async { final cache = await _newCacheFromStore(store, maxEntries: maxEntries); - return MetadataCache(cache); + return MetadataCache(cache, maxEntries: maxEntries); } - Future put(String key, Uint8List data) async { + Future put(String key, Uint8List data) async { + final isFull = await this.isFull; + if (isFull) { + logger.d('Cache is full, not putting $key in metadata cache'); + return false; + } + logger.d('Putting $key in metadata cache'); - return _cache.put(key, data); + await _cache.put(key, data); + return true; } Future get(String key) async { @@ -48,16 +57,29 @@ class MetadataCache { return _cache.keys; } + Future get isFull async { + final size = await this.size; + final isFull = size >= _maxEntries; + logger.d('Cache is full: $isFull - size: $size, max: $defaultMaxEntries'); + return isFull; + } + + Future get size async { + return _cache.size; + } + static Future> _newCacheFromStore( CacheStore store, { - int maxEntries = defaultMaxEntries, - }) { + required int maxEntries, + }) async { + logger.d('Creating metadata cache with max entries: $maxEntries'); + return store.cache( name: defaultCacheName, maxEntries: maxEntries, // See: https://pub.dev/packages/stash#eviction-policies - evictionPolicy: const LfuEvictionPolicy(), + evictionPolicy: null, ); } } From db517dbcc591c04be53fb7367c8b48402ee7ecf0 Mon Sep 17 00:00:00 2001 From: Mati Date: Thu, 25 May 2023 12:53:11 -0300 Subject: [PATCH 22/28] test(metadata cache): unit tests custom no-eviction PE-3786 --- test/utils/metadata_cache_test.dart | 59 +++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/test/utils/metadata_cache_test.dart b/test/utils/metadata_cache_test.dart index c2a436e31a..e261844b3b 100644 --- a/test/utils/metadata_cache_test.dart +++ b/test/utils/metadata_cache_test.dart @@ -37,7 +37,36 @@ void main() { expect(keys, ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']); }); - test('follows the eviction policy: Least Frequently Used (LFU)', () async { + test('size indicates the number of entries in the cache', () async { + final metadataCache = await MetadataCache.fromCacheStore( + await newMemoryCacheStore(), + maxEntries: 10, + ); + + final mockData = generateMockData(1)[0]; + + await metadataCache.put('0', mockData); + + expect(await metadataCache.size, 1); + }); + + test('isFull indicates whether the cache is at max capacity', () async { + final metadataCache = await MetadataCache.fromCacheStore( + await newMemoryCacheStore(), + maxEntries: 10, + ); + + final mockData = generateMockData(10); + + for (int i = 0; i < mockData.length; i++) { + final wasPut = await metadataCache.put(i.toString(), mockData[i]); + expect(wasPut, true); + } + + expect(await metadataCache.isFull, true); + }); + + test('follows the eviction policy: no eviction', () async { // Refer to https://pub.dev/packages/stash#eviction-policies for more info /// LfuEvictionPolicy LFU (least-frequently used) policy counts how often /// an entry is used. Those that are least often used are discarded first. @@ -53,29 +82,29 @@ void main() { final mockData = generateMockData(10); for (int i = 0; i < mockData.length; i++) { - await metadataCache.put(i.toString(), mockData[i]); - - if (i != 0) { - // These becomes the most frequently used items - /// and zero becomes the least frequently used - await metadataCache.get(i.toString()); - await metadataCache.get(i.toString()); - } + final wasPut = await metadataCache.put(i.toString(), mockData[i]); + expect(wasPut, true); } - expect(await metadataCache.get('0'), isNotNull); + expect( + await metadataCache.keys, + ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'], + ); - await metadataCache.put('eleventh-item', Uint8List.fromList([1])); + final wasEleventhItemPut = await metadataCache.put( + 'eleventh-item', + Uint8List.fromList([1]), + ); + expect(wasEleventhItemPut, false); expect( - await metadataCache.get('0'), + await metadataCache.get('eleventh-item'), null, ); - final keys = await metadataCache.keys; expect( - keys, - ['1', '2', '3', '4', '5', '6', '7', '8', '9', 'eleventh-item'], + await metadataCache.keys, + ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'], ); }); From 3ce2679bccfdf8ab72b4af4d3c17b9913827ffe3 Mon Sep 17 00:00:00 2001 From: Mati Date: Thu, 25 May 2023 13:31:32 -0300 Subject: [PATCH 23/28] feat(metadata cache): quits unnecessary logs PE-3786 --- lib/utils/metadata_cache.dart | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/utils/metadata_cache.dart b/lib/utils/metadata_cache.dart index ad99458cbf..cc79ce4c45 100644 --- a/lib/utils/metadata_cache.dart +++ b/lib/utils/metadata_cache.dart @@ -9,8 +9,9 @@ const defaultCacheName = 'metadata-cache'; class MetadataCache { final Cache _cache; final int _maxEntries; + bool _isFull = false; - const MetadataCache(this._cache, {int maxEntries = defaultMaxEntries}) + MetadataCache(this._cache, {int maxEntries = defaultMaxEntries}) : _maxEntries = maxEntries; static Future fromCacheStore( @@ -24,7 +25,6 @@ class MetadataCache { Future put(String key, Uint8List data) async { final isFull = await this.isFull; if (isFull) { - logger.d('Cache is full, not putting $key in metadata cache'); return false; } @@ -58,9 +58,18 @@ class MetadataCache { } Future get isFull async { + if (_isFull) { + return true; + } + final size = await this.size; final isFull = size >= _maxEntries; - logger.d('Cache is full: $isFull - size: $size, max: $defaultMaxEntries'); + _isFull = isFull; + + if (isFull) { + logger.d('Metadata cache is full and will not accept new entries'); + } + return isFull; } From d657c23426684098fff804be36cafa139212edf1 Mon Sep 17 00:00:00 2001 From: Mati Date: Thu, 25 May 2023 13:36:02 -0300 Subject: [PATCH 24/28] feat(arweave service): replaves warning with error level logging for catch blocks PE-3786 --- lib/services/arweave/arweave_service.dart | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/services/arweave/arweave_service.dart b/lib/services/arweave/arweave_service.dart index fcf171268a..15400bcaa7 100644 --- a/lib/services/arweave/arweave_service.dart +++ b/lib/services/arweave/arweave_service.dart @@ -313,12 +313,12 @@ class ArweaveService { // If there are errors in parsing the entity, ignore it. } on EntityTransactionParseException catch (parseException) { - logger.w( + logger.e( 'Failed to parse transaction ' 'with id ${parseException.transactionId}', ); } on GatewayError catch (fetchException) { - logger.w( + logger.e( 'Failed to fetch entity data with the exception ${fetchException.runtimeType}' 'for transaction ${transaction.id}, ' 'with status ${fetchException.statusCode} ' @@ -507,7 +507,7 @@ class ArweaveService { // If there's an error parsing the drive entity, just ignore it. } on EntityTransactionParseException catch (parseException) { - logger.w( + logger.e( 'Failed to parse transaction ' 'with id ${parseException.transactionId}', ); @@ -515,7 +515,7 @@ class ArweaveService { } return drivesWithKey; } catch (e, stacktrace) { - logger.w( + logger.e( 'An error occurred when getting the unique user drive entities.' ' Exception: ${e.toString()} stacktrace: ${stacktrace.toString()}', ); @@ -569,7 +569,7 @@ class ArweaveService { return await DriveEntity.fromTransaction( fileTx, _crypto, fileDataRes.bodyBytes, driveKey); } on EntityTransactionParseException catch (parseException) { - logger.w( + logger.e( 'Failed to parse transaction ' 'with id ${parseException.transactionId}', ); @@ -731,7 +731,7 @@ class ArweaveService { crypto: _crypto, ); } on EntityTransactionParseException catch (parseException) { - logger.w( + logger.e( 'Failed to parse transaction ' 'with id ${parseException.transactionId}', ); @@ -848,7 +848,7 @@ class ArweaveService { try { await Future.wait(confirmationFutures); } catch (e) { - logger.w('Error getting transactions confirmations on exception: $e'); + logger.e('Error getting transactions confirmations on exception: $e'); rethrow; } From 4f14c2e8b9f4ec4ceafb2af286f06204281a7ca7 Mon Sep 17 00:00:00 2001 From: Mati Date: Thu, 25 May 2023 13:43:22 -0300 Subject: [PATCH 25/28] feat(metadata cache): improve logging PE-3786 --- lib/utils/metadata_cache.dart | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/lib/utils/metadata_cache.dart b/lib/utils/metadata_cache.dart index cc79ce4c45..5d5e440036 100644 --- a/lib/utils/metadata_cache.dart +++ b/lib/utils/metadata_cache.dart @@ -9,7 +9,6 @@ const defaultCacheName = 'metadata-cache'; class MetadataCache { final Cache _cache; final int _maxEntries; - bool _isFull = false; MetadataCache(this._cache, {int maxEntries = defaultMaxEntries}) : _maxEntries = maxEntries; @@ -23,13 +22,17 @@ class MetadataCache { } Future put(String key, Uint8List data) async { - final isFull = await this.isFull; - if (isFull) { + if (await isFull) { return false; } logger.d('Putting $key in metadata cache'); await _cache.put(key, data); + + if (await isFull) { + logger.i('Metadata cache is now full and will not accept new entries'); + } + return true; } @@ -58,17 +61,8 @@ class MetadataCache { } Future get isFull async { - if (_isFull) { - return true; - } - final size = await this.size; final isFull = size >= _maxEntries; - _isFull = isFull; - - if (isFull) { - logger.d('Metadata cache is full and will not accept new entries'); - } return isFull; } From 1bf800d7777c3d27b823efbfa31bef0c129d97e9 Mon Sep 17 00:00:00 2001 From: Mati Date: Thu, 1 Jun 2023 14:58:25 -0300 Subject: [PATCH 26/28] feat(metadata cache): sets a narrower limit; improves error handling PE-3786 --- lib/utils/metadata_cache.dart | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/lib/utils/metadata_cache.dart b/lib/utils/metadata_cache.dart index 5d5e440036..4cd30c214d 100644 --- a/lib/utils/metadata_cache.dart +++ b/lib/utils/metadata_cache.dart @@ -3,7 +3,7 @@ import 'dart:typed_data'; import 'package:ardrive/utils/logger/logger.dart'; import 'package:stash/stash_api.dart'; -const defaultMaxEntries = 20000; +const defaultMaxEntries = 550; const defaultCacheName = 'metadata-cache'; class MetadataCache { @@ -26,8 +26,14 @@ class MetadataCache { return false; } - logger.d('Putting $key in metadata cache'); - await _cache.put(key, data); + // FIXME: check for quota before attempting to write to cache + try { + logger.d('Putting $key in metadata cache'); + await _cache.putIfAbsent(key, data); + } catch (e, s) { + logger.e('Failed to put $key in metadata cache', e, s); + return false; + } if (await isFull) { logger.i('Metadata cache is now full and will not accept new entries'); @@ -37,13 +43,18 @@ class MetadataCache { } Future get(String key) async { - final value = await _cache.get(key); - if (value != null) { - logger.d('Cache hit for $key in metadata cache'); - } else { - logger.d('Cache miss for $key in metadata cache'); + try { + final value = await _cache.get(key); + if (value != null) { + logger.d('Cache hit for $key in metadata cache'); + } else { + logger.d('Cache miss for $key in metadata cache'); + } + return value; + } catch (e, s) { + logger.e('Failed to get $key from metadata cache', e, s); + return null; } - return value; } Future remove(String key) async { From 9e9b7da22e73a47a170dda8ccc69a4f9eb6c2cb1 Mon Sep 17 00:00:00 2001 From: Mati Date: Mon, 12 Jun 2023 12:57:52 -0300 Subject: [PATCH 27/28] feat(version): before-release version bump PE-4001 --- pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubspec.yaml b/pubspec.yaml index 68b22c3de7..10b92c5513 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -3,7 +3,7 @@ description: Secure, permanent storage publish_to: 'none' -version: 2.2.0 +version: 2.3.0 environment: sdk: '>=2.18.5 <3.0.0' From 80c9eb14a7d1fb6bc9017618eec5289775fdfeaa Mon Sep 17 00:00:00 2001 From: Mati Date: Mon, 12 Jun 2023 12:58:14 -0300 Subject: [PATCH 28/28] feat(release notes): before-release android release notes PE-4001 --- android/fastlane/metadata/android/en-US/changelogs/46.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 android/fastlane/metadata/android/en-US/changelogs/46.txt diff --git a/android/fastlane/metadata/android/en-US/changelogs/46.txt b/android/fastlane/metadata/android/en-US/changelogs/46.txt new file mode 100644 index 0000000000..07aebe6b4f --- /dev/null +++ b/android/fastlane/metadata/android/en-US/changelogs/46.txt @@ -0,0 +1 @@ +- Utilize cached metadata for Snapshot authoring