From 9720a406507e97e1a53fd16c811cb6e246d1a6f5 Mon Sep 17 00:00:00 2001 From: Edison Zhang Date: Mon, 7 Oct 2024 17:37:38 -0700 Subject: [PATCH 1/6] added comments for migration place in SQLiteStorageAdapter --- .../datastore/storage/sqlite/SQLiteStorageAdapter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/SQLiteStorageAdapter.java b/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/SQLiteStorageAdapter.java index ed876ca4e..1c8d32f66 100644 --- a/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/SQLiteStorageAdapter.java +++ b/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/SQLiteStorageAdapter.java @@ -934,6 +934,9 @@ private Completable updateModels() { Objects.requireNonNull(databaseConnectionHandle); sqliteStorageHelper.update(databaseConnectionHandle, oldVersion, newVersion); } else { + // We only need to do the model migration here because + // the current implementation of sqliteStorageHelper.update will drop all existing tables and recreate tables with new schemas, + // However, this might be changed in the future LOG.debug("Database up to date. Checking ModelMetadata."); new ModelMigrations(databaseConnectionHandle, modelsProvider).apply(); } From a28ad3e6909557efeb2b3b5e4a89cb6fdcf57dab Mon Sep 17 00:00:00 2001 From: Edison Zhang Date: Mon, 7 Oct 2024 17:46:29 -0700 Subject: [PATCH 2/6] changed log before migration --- .../datastore/storage/sqlite/SQLiteStorageAdapter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/SQLiteStorageAdapter.java b/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/SQLiteStorageAdapter.java index 1c8d32f66..f604ce800 100644 --- a/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/SQLiteStorageAdapter.java +++ b/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/SQLiteStorageAdapter.java @@ -937,7 +937,7 @@ private Completable updateModels() { // We only need to do the model migration here because // the current implementation of sqliteStorageHelper.update will drop all existing tables and recreate tables with new schemas, // However, this might be changed in the future - LOG.debug("Database up to date. Checking ModelMetadata."); + LOG.debug("Database up to date. Checking System Models."); new ModelMigrations(databaseConnectionHandle, modelsProvider).apply(); } } From 24a24c204a1b5a5a79adb38ade979ace3ec571c2 Mon Sep 17 00:00:00 2001 From: Edison Zhang Date: Mon, 7 Oct 2024 18:29:19 -0700 Subject: [PATCH 3/6] added AddSyncExpressionToLastSyncMetadata --- .../AddSyncExpressionToLastSyncMetadata.java | 44 +++++++++++++++++++ .../sqlite/migrations/ModelMigrations.java | 1 + 2 files changed, 45 insertions(+) create mode 100644 aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/migrations/AddSyncExpressionToLastSyncMetadata.java diff --git a/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/migrations/AddSyncExpressionToLastSyncMetadata.java b/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/migrations/AddSyncExpressionToLastSyncMetadata.java new file mode 100644 index 000000000..e8074bc77 --- /dev/null +++ b/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/migrations/AddSyncExpressionToLastSyncMetadata.java @@ -0,0 +1,44 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amplifyframework.datastore.storage.sqlite.migrations; + +import android.database.sqlite.SQLiteDatabase; + +import com.amplifyframework.core.Amplify; +import com.amplifyframework.core.category.CategoryType; +import com.amplifyframework.core.model.ModelProvider; +import com.amplifyframework.logging.Logger; + +public class AddSyncExpressionToLastSyncMetadata implements ModelMigration{ + private static final Logger LOG = Amplify.Logging.logger(CategoryType.DATASTORE, "amplify:aws-datastore"); + private final SQLiteDatabase database; + private final ModelProvider modelProvider; + + /** + * Constructor for the migration class. + * @param database Connection to the SQLite database. + * @param modelProvider The model provider. + */ + public AddSyncExpressionToLastSyncMetadata(SQLiteDatabase database, ModelProvider modelProvider) { + this.database = database; + this.modelProvider = modelProvider; + } + + @Override + public void apply() { + LOG.info("AddSyncExpressionToLastSyncMetadata is applied"); + } +} diff --git a/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/migrations/ModelMigrations.java b/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/migrations/ModelMigrations.java index 6afe6509b..68ffd7a22 100644 --- a/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/migrations/ModelMigrations.java +++ b/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/migrations/ModelMigrations.java @@ -37,6 +37,7 @@ public class ModelMigrations { public ModelMigrations(SQLiteDatabase databaseConnectionHandle, ModelProvider modelsProvider) { List migrationClasses = new ArrayList<>(); migrationClasses.add(new AddModelNameToModelMetadataKey(databaseConnectionHandle, modelsProvider)); + migrationClasses.add(new AddSyncExpressionToLastSyncMetadata(databaseConnectionHandle, modelsProvider)); this.modelMigrations = Immutable.of(migrationClasses); } From 6a3f061a11b422a4fd69f37946551e6609026942 Mon Sep 17 00:00:00 2001 From: Edison Zhang Date: Tue, 8 Oct 2024 08:57:49 -0700 Subject: [PATCH 4/6] DB migration (MVP) --- .../AddSyncExpressionToLastSyncMetadata.java | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/migrations/AddSyncExpressionToLastSyncMetadata.java b/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/migrations/AddSyncExpressionToLastSyncMetadata.java index e8074bc77..5b353dc14 100644 --- a/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/migrations/AddSyncExpressionToLastSyncMetadata.java +++ b/aws-datastore/src/main/java/com/amplifyframework/datastore/storage/sqlite/migrations/AddSyncExpressionToLastSyncMetadata.java @@ -15,17 +15,20 @@ package com.amplifyframework.datastore.storage.sqlite.migrations; +import android.database.Cursor; import android.database.sqlite.SQLiteDatabase; import com.amplifyframework.core.Amplify; import com.amplifyframework.core.category.CategoryType; import com.amplifyframework.core.model.ModelProvider; import com.amplifyframework.logging.Logger; +import com.amplifyframework.util.Wrap; public class AddSyncExpressionToLastSyncMetadata implements ModelMigration{ private static final Logger LOG = Amplify.Logging.logger(CategoryType.DATASTORE, "amplify:aws-datastore"); private final SQLiteDatabase database; private final ModelProvider modelProvider; + private final String newSyncExpColumnName = "syncExpression"; /** * Constructor for the migration class. @@ -39,6 +42,41 @@ public AddSyncExpressionToLastSyncMetadata(SQLiteDatabase database, ModelProvide @Override public void apply() { - LOG.info("AddSyncExpressionToLastSyncMetadata is applied"); + if(!needsMigration()) { + LOG.debug("No LastSyncMetadata migration needed."); + return; + } + addNewSyncExpColumnName(); + } + + /** + * Existing rows in LasySyncMetadata will have 'null' for ${newSyncExpColumnName} value, + * until the next sync/hydrate operation + */ + private void addNewSyncExpColumnName() { + try { + database.beginTransaction(); + final String addColumnSql = "ALTER TABLE LastSyncMetadata ADD COLUMN " + + newSyncExpColumnName + " TEXT"; + database.execSQL(addColumnSql); + database.setTransactionSuccessful(); + LOG.debug("Successfully upgraded LastSyncMetadata table with new field: " + newSyncExpColumnName); + } finally { + if (database.inTransaction()) { + database.endTransaction(); + } + } + } + + private boolean needsMigration() { + final String checkColumnSql = "SELECT COUNT(*) FROM pragma_table_info('LastSyncMetadata') " + + "WHERE name=" + Wrap.inSingleQuotes(newSyncExpColumnName); + try (Cursor queryResults = database.rawQuery(checkColumnSql, new String[]{})) { + if (queryResults.moveToNext()) { + int recordNum = queryResults.getInt(0); + return recordNum==0; // needs to be upgraded if there's no column named ${newSyncExpColumnName} + } + } + return false; } } From 455f879e93022e9906c4787d2d718a6d69eb4cca Mon Sep 17 00:00:00 2001 From: Edison Zhang Date: Tue, 8 Oct 2024 10:47:07 -0700 Subject: [PATCH 5/6] add new field to LastSyncMetadata model (MVP: pre-deserialized Query Predicate) --- .../syncengine/LastSyncMetadata.java | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/aws-datastore/src/main/java/com/amplifyframework/datastore/syncengine/LastSyncMetadata.java b/aws-datastore/src/main/java/com/amplifyframework/datastore/syncengine/LastSyncMetadata.java index 41ceaba79..c4cbda49d 100644 --- a/aws-datastore/src/main/java/com/amplifyframework/datastore/syncengine/LastSyncMetadata.java +++ b/aws-datastore/src/main/java/com/amplifyframework/datastore/syncengine/LastSyncMetadata.java @@ -27,7 +27,7 @@ import java.util.UUID; /** - * Metadata about the last time that a model class was sync'd with AppSync backend. + * Metadata about the last time that a model class was sync'd with AppSync backend using a certain syncExpression. * This metadata is persisted locally as a system model. This metadata is inspected * whenever the Sync Engine starts up. The system consider the value of * {@link LastSyncMetadata#getLastSyncTime()} to decide whether or not it should @@ -39,13 +39,15 @@ public final class LastSyncMetadata implements Model { private final @ModelField(targetType = "String", isRequired = true) String modelClassName; private final @ModelField(targetType = "AWSTimestamp", isRequired = true) Long lastSyncTime; private final @ModelField(targetType = "String", isRequired = true) String lastSyncType; + private final @ModelField(targetType = "String") String syncExpression; @SuppressWarnings("checkstyle:ParameterName") // The field is named "id" in the model; keep it consistent - private LastSyncMetadata(String id, String modelClassName, Long lastSyncTime, SyncType syncType) { + private LastSyncMetadata(String id, String modelClassName, Long lastSyncTime, SyncType syncType, @Nullable String syncExpression) { this.id = id; this.modelClassName = modelClassName; this.lastSyncTime = lastSyncTime; this.lastSyncType = syncType.name(); + this.syncExpression = syncExpression; } /** @@ -57,9 +59,9 @@ private LastSyncMetadata(String id, String modelClassName, Long lastSyncTime, Sy * @return {@link LastSyncMetadata} for the model class */ public static LastSyncMetadata baseSyncedAt(@NonNull String modelClassName, - @Nullable long lastSyncTime) { + @Nullable long lastSyncTime, @Nullable String syncExpression) { Objects.requireNonNull(modelClassName); - return create(modelClassName, lastSyncTime, SyncType.BASE); + return create(modelClassName, lastSyncTime, SyncType.BASE, syncExpression); } /** @@ -70,9 +72,9 @@ public static LastSyncMetadata baseSyncedAt(@NonNull String mo * @return {@link LastSyncMetadata} for the model class */ static LastSyncMetadata deltaSyncedAt(@NonNull String modelClassName, - @Nullable long lastSyncTime) { + @Nullable long lastSyncTime, @Nullable String syncExpression) { Objects.requireNonNull(modelClassName); - return create(modelClassName, lastSyncTime, SyncType.DELTA); + return create(modelClassName, lastSyncTime, SyncType.DELTA, syncExpression); } /** @@ -84,7 +86,7 @@ static LastSyncMetadata deltaSyncedAt(@NonNull String modelCla */ public static LastSyncMetadata neverSynced(@NonNull String modelClassName) { Objects.requireNonNull(modelClassName); - return create(modelClassName, null, SyncType.BASE); + return create(modelClassName, null, SyncType.BASE, null); } /** @@ -97,9 +99,9 @@ public static LastSyncMetadata neverSynced(@NonNull String mod */ @SuppressWarnings("WeakerAccess") static LastSyncMetadata create( - @NonNull String modelClassName, @Nullable Long lastSyncTime, @NonNull SyncType syncType) { + @NonNull String modelClassName, @Nullable Long lastSyncTime, @NonNull SyncType syncType, @Nullable String syncExpression) { Objects.requireNonNull(modelClassName); - return new LastSyncMetadata(hash(modelClassName), modelClassName, lastSyncTime, syncType); + return new LastSyncMetadata(hash(modelClassName), modelClassName, lastSyncTime, syncType, syncExpression); } @NonNull @@ -144,6 +146,14 @@ public String getLastSyncType() { return lastSyncType; } + /** + * Returns the sync expression being used in the last sync + * @return A serialized sync expression + */ + public String getSyncExpression() { + return this.syncExpression; + } + /** * Computes a stable hash for a model class, by its name. * Since {@link Model}s have to have unique IDs, we need an ID for this class. @@ -175,6 +185,9 @@ public boolean equals(Object thatObject) { if (!ObjectsCompat.equals(lastSyncType, that.lastSyncType)) { return false; } + if (!ObjectsCompat.equals(syncExpression, that.syncExpression)) { + return false; + } return ObjectsCompat.equals(lastSyncTime, that.lastSyncTime); } @@ -184,6 +197,7 @@ public int hashCode() { result = 31 * result + modelClassName.hashCode(); result = 31 * result + lastSyncTime.hashCode(); result = 31 * result + lastSyncType.hashCode(); + result = 31 * result + syncExpression.hashCode(); return result; } @@ -194,6 +208,7 @@ public String toString() { ", modelClassName='" + modelClassName + '\'' + ", lastSyncTime=" + lastSyncTime + ", lastSyncType=" + lastSyncType + + ", syncExpression" + syncExpression + '}'; } } From 1fc5f094a5d664d4c5e07ac9712765271e67a9cc Mon Sep 17 00:00:00 2001 From: Edison Zhang Date: Tue, 8 Oct 2024 11:05:44 -0700 Subject: [PATCH 6/6] updated signature of saveLastDelta/BaseSyncTime and added TODO in SyncTimeRegistry&SyncProcessor --- .../datastore/syncengine/SyncProcessor.java | 8 ++++++-- .../datastore/syncengine/SyncTimeRegistry.java | 14 ++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/aws-datastore/src/main/java/com/amplifyframework/datastore/syncengine/SyncProcessor.java b/aws-datastore/src/main/java/com/amplifyframework/datastore/syncengine/SyncProcessor.java index 43bd39e08..21a8c59bb 100644 --- a/aws-datastore/src/main/java/com/amplifyframework/datastore/syncengine/SyncProcessor.java +++ b/aws-datastore/src/main/java/com/amplifyframework/datastore/syncengine/SyncProcessor.java @@ -182,6 +182,9 @@ Completable hydrate() { private Completable createHydrationTask(ModelSchema schema) { ModelSyncMetricsAccumulator metricsAccumulator = new ModelSyncMetricsAccumulator(schema.getName()); + //TODO: QueryPredicate currentPredicate = this.queryPredicateProvider.getPredicate(schema.getName()); + //TODO deserialize current predicate to deserializedCurrentPredicate + //TODO: pass deserializedCurrentPredicate to lookupLastSyncTime as the second parameter return syncTimeRegistry.lookupLastSyncTime(schema.getName()) .map(this::filterOutOldSyncTimes) // And for each, perform a sync. The network response will contain an Iterable> @@ -201,9 +204,10 @@ private Completable createHydrationTask(ModelSchema schema) { .toSingle(() -> lastSyncTime.exists() ? SyncType.DELTA : SyncType.BASE); }) .flatMapCompletable(syncType -> { + //TODO: pass the deserializedCurrentPredicate to saveLastDelta/BaseSync as the third parameter Completable syncTimeSaveCompletable = SyncType.DELTA.equals(syncType) ? - syncTimeRegistry.saveLastDeltaSyncTime(schema.getName(), SyncTime.now()) : - syncTimeRegistry.saveLastBaseSyncTime(schema.getName(), SyncTime.now()); + syncTimeRegistry.saveLastDeltaSyncTime(schema.getName(), SyncTime.now(), null) : + syncTimeRegistry.saveLastBaseSyncTime(schema.getName(), SyncTime.now(), null); return syncTimeSaveCompletable.andThen(Completable.fromAction(() -> Amplify.Hub.publish( HubChannel.DATASTORE, metricsAccumulator.toModelSyncedEvent(syncType).toHubEvent() diff --git a/aws-datastore/src/main/java/com/amplifyframework/datastore/syncengine/SyncTimeRegistry.java b/aws-datastore/src/main/java/com/amplifyframework/datastore/syncengine/SyncTimeRegistry.java index 1a17f1763..a20a35f52 100644 --- a/aws-datastore/src/main/java/com/amplifyframework/datastore/syncengine/SyncTimeRegistry.java +++ b/aws-datastore/src/main/java/com/amplifyframework/datastore/syncengine/SyncTimeRegistry.java @@ -40,6 +40,7 @@ final class SyncTimeRegistry { this.localStorageAdapter = localStorageAdapter; } + //TODO: add the second argument: current sync expression here Single lookupLastSyncTime(@NonNull String modelClassName) { return Single.create(emitter -> { QueryPredicate hasMatchingModelClassName = QueryField.field("modelClassName").eq(modelClassName); @@ -47,6 +48,9 @@ Single lookupLastSyncTime(@NonNull String modelClassName) { localStorageAdapter.query(LastSyncMetadata.class, Where.matches(hasMatchingModelClassName), results -> { try { LastSyncMetadata syncMetadata = extractSingleResult(modelClassName, results); + //TODO: syncMetadata should contain the previous serialized sync expression + //TODO: compare the previous sync expression with the current one + //TODO: emmit SyncTime.never() if different emitter.onSuccess(SyncTime.from(syncMetadata.getLastSyncTime())); } catch (DataStoreException queryResultFailure) { emitter.onError(queryResultFailure); @@ -55,9 +59,10 @@ Single lookupLastSyncTime(@NonNull String modelClassName) { }); } - Completable saveLastDeltaSyncTime(@NonNull String modelClassName, @Nullable SyncTime syncTime) { + //TODO: change the name to saveLastDeltaSync, and add the second argument for sync expression + Completable saveLastDeltaSyncTime(@NonNull String modelClassName, @Nullable SyncTime syncTime, @Nullable String syncExpression) { LastSyncMetadata metadata = syncTime != null && syncTime.exists() ? - LastSyncMetadata.deltaSyncedAt(modelClassName, syncTime.toLong()) : + LastSyncMetadata.deltaSyncedAt(modelClassName, syncTime.toLong(), syncExpression) : LastSyncMetadata.neverSynced(modelClassName); return Completable.create(emitter -> @@ -71,9 +76,10 @@ Completable saveLastDeltaSyncTime(@NonNull String modelClassName, @Nullable Sync ); } - Completable saveLastBaseSyncTime(@NonNull String modelClassName, @Nullable SyncTime syncTime) { + //TODO: change the name to saveLastDeltaSync, and add the second argument for sync expression + Completable saveLastBaseSyncTime(@NonNull String modelClassName, @Nullable SyncTime syncTime, @Nullable String syncExpression) { LastSyncMetadata metadata = syncTime != null && syncTime.exists() ? - LastSyncMetadata.baseSyncedAt(modelClassName, syncTime.toLong()) : + LastSyncMetadata.baseSyncedAt(modelClassName, syncTime.toLong(), syncExpression) : LastSyncMetadata.neverSynced(modelClassName); return Completable.create(emitter ->