Merge pull request #1545 from wmathurin/json1

Support for json1 indexes

libs/SmartStore/SmartStore.xcodeproj/project.pbxproj

@@ -31,6 +31,7 @@
 		4F06AFF11C49D53D00F70798 /* SmartStore.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = CE4CE2B91C0E4581009F6029 /* SmartStore.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
 		4F883C761C16279F007D4BAE /* SalesforceSDKManagerWithSmartStore.h in Headers */ = {isa = PBXBuildFile; fileRef = 4F883C751C16279F007D4BAE /* SalesforceSDKManagerWithSmartStore.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		4F883C7B1C1627BD007D4BAE /* SalesforceSDKManagerWithSmartStore.m in Sources */ = {isa = PBXBuildFile; fileRef = 4F883C7A1C1627BD007D4BAE /* SalesforceSDKManagerWithSmartStore.m */; };
+		4F99B53C1CEA81A6007BC4D2 /* SFSmartStoreLoadTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 4F99B53B1CEA81A6007BC4D2 /* SFSmartStoreLoadTests.m */; };
 		4FEE44851BFD5CCC00F09C43 /* SFSmartSqlTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 4F9601541BFD33B30022F021 /* SFSmartSqlTests.m */; };
 		4FEE44861BFD5CD200F09C43 /* SFSmartStoreAlterTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 4F9601561BFD33B30022F021 /* SFSmartStoreAlterTests.m */; };
 		4FEE44871BFD5CD500F09C43 /* SFSmartStoreFullTextSearchSpeedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 4F9601581BFD33B30022F021 /* SFSmartStoreFullTextSearchSpeedTests.m */; };
@@ -257,6 +258,8 @@
 		4F96FC4E1BFD31EF0022F021 /* SmartStore-Static-iOS-Debug.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = "SmartStore-Static-iOS-Debug.xcconfig"; path = "Configuration/SmartStore-Static-iOS-Debug.xcconfig"; sourceTree = "<group>"; };
 		4F96FC4F1BFD31EF0022F021 /* SmartStore-Static-iOS-Release.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = "SmartStore-Static-iOS-Release.xcconfig"; path = "Configuration/SmartStore-Static-iOS-Release.xcconfig"; sourceTree = "<group>"; };
 		4F96FC501BFD31EF0022F021 /* SmartStore-Test.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = "SmartStore-Test.xcconfig"; path = "Configuration/SmartStore-Test.xcconfig"; sourceTree = "<group>"; };
+		4F99B53B1CEA81A6007BC4D2 /* SFSmartStoreLoadTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SFSmartStoreLoadTests.m; sourceTree = "<group>"; };
+		4F99B5401CEA81C6007BC4D2 /* SFSmartStoreLoadTests.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SFSmartStoreLoadTests.h; sourceTree = "<group>"; };
 		4FFEE6341BFE98B600B7AA8A /* SmartStoreTests-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = "SmartStoreTests-Info.plist"; path = "SmartStoreTests/SmartStoreTests-Info.plist"; sourceTree = SOURCE_ROOT; };
 		8289177A1C52B705002F9981 /* FMDatabase.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = FMDatabase.h; path = ../../../../external/fmdb/src/fmdb/FMDatabase.h; sourceTree = "<group>"; };
 		8289177B1C52B705002F9981 /* FMDatabase.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = FMDatabase.m; path = ../../../../external/fmdb/src/fmdb/FMDatabase.m; sourceTree = "<group>"; };
@@ -480,10 +483,12 @@
 		CEAAAE5E195911E600CBBFE9 /* SmartStoreTests */ = {
 			isa = PBXGroup;
 			children = (
+				4F99B5401CEA81C6007BC4D2 /* SFSmartStoreLoadTests.h */,
 				4F9601531BFD33B30022F021 /* SFSmartSqlTests.h */,
 				4F9601541BFD33B30022F021 /* SFSmartSqlTests.m */,
 				4F9601551BFD33B30022F021 /* SFSmartStoreAlterTests.h */,
 				4F9601561BFD33B30022F021 /* SFSmartStoreAlterTests.m */,
+				4F99B53B1CEA81A6007BC4D2 /* SFSmartStoreLoadTests.m */,
 				4F9601571BFD33B30022F021 /* SFSmartStoreFullTextSearchSpeedTests.h */,
 				4F9601581BFD33B30022F021 /* SFSmartStoreFullTextSearchSpeedTests.m */,
 				4F9601591BFD33B30022F021 /* SFSmartStoreFullTextSearchTests.h */,
@@ -884,6 +889,7 @@
 				4FEE44891BFD5CDC00F09C43 /* SFSmartStoreTestCase.m in Sources */,
 				4FEE44851BFD5CCC00F09C43 /* SFSmartSqlTests.m in Sources */,
 				4FEE44861BFD5CD200F09C43 /* SFSmartStoreAlterTests.m in Sources */,
+				4F99B53C1CEA81A6007BC4D2 /* SFSmartStoreLoadTests.m in Sources */,
 				4FEE44881BFD5CD900F09C43 /* SFSmartStoreFullTextSearchTests.m in Sources */,
 				4FEE448A1BFD5CE500F09C43 /* SFSmartStoreTests.m in Sources */,
 				4FEE44871BFD5CD500F09C43 /* SFSmartStoreFullTextSearchSpeedTests.m in Sources */,

libs/SmartStore/SmartStore/Classes/SFAlterSoupLongOperation.m

@@ -219,6 +219,12 @@ - (void) copyTable
         for (NSString* keptPath in keptPaths) {
             SFSoupIndex* oldIndexSpec = mapOldSpecs[keptPath];
             SFSoupIndex* newIndexSpec = mapNewSpecs[keptPath];
+
+            if (newIndexSpec.columnType == nil) {
+                // we are now using json1, there is no column to populate
+                continue;
+            }
+
             if ([oldIndexSpec.columnType isEqualToString:newIndexSpec.columnType]) {
                 [oldColumns addObject:oldIndexSpec.columnName];
                 [newColumns addObject:newIndexSpec.columnName];

libs/SmartStore/SmartStore/Classes/SFSmartSqlHelper.m

@@ -113,6 +113,13 @@ - (NSString*) convertSmartSql:(NSString*)smartSql withStore:(SFSmartStore*) stor
         }
     }
 
+    // With json1 support, the column name could be an expression of the form json_extract(soup, '$.x.y.z')
+    // We can't have TABLE_x.json_extract(soup, ...) in the sql query
+    // Instead we should have json_extract(TABLE_x.soup, ...)
+    NSError *error = nil;
+    NSRegularExpression *regex = [NSRegularExpression regularExpressionWithPattern:@"(TABLE_[0-9]+)\\.json_extract\\(soup" options:0 error:&error];
+    [regex replaceMatchesInString:sql options:0 range:NSMakeRange(0, [sql length]) withTemplate:@"json_extract($1.soup"];
+
     return sql;
 }
 

libs/SmartStore/SmartStore/Classes/SFSmartStore.h

@@ -89,6 +89,12 @@ extern NSString *const STATUS_COL;
 extern NSString *const SOUP_ENTRY_ID;
 extern NSString *const SOUP_LAST_MODIFIED_DATE;
 
+/*
+ Support for explain query plan
+ */
+extern NSString *const EXPLAIN_SQL;
+extern NSString *const EXPLAIN_ARGS;
+extern NSString *const EXPLAIN_ROWS;
 
 @class FMDatabaseQueue;
 @class SFQuerySpec;
@@ -125,6 +131,16 @@ extern NSString *const SOUP_LAST_MODIFIED_DATE;
  */
 @property (nonatomic, strong) SFUserAccount *user;
 
+/**
+ Flag to cause explain plan to be captured for every query
+ */
+@property (nonatomic, assign) BOOL captureExplainQueryPlan;
+
+/**
+ Dictionary with results of last explain query plan
+ */
+@property (nonatomic, strong) NSDictionary *lastExplainQueryPlan;
+
 /**
  Use this method to obtain a shared store instance with a particular name for the current user.
  

libs/SmartStore/SmartStore/Classes/SFSmartStore.m

@@ -97,13 +97,12 @@
 NSString *const SOUP_ENTRY_ID = @"_soupEntryId";
 NSString *const SOUP_LAST_MODIFIED_DATE = @"_soupLastModifiedDate";
 
-@implementation SFSmartStore
+// Explain support
+NSString *const EXPLAIN_SQL = @"sql";
+NSString *const EXPLAIN_ARGS = @"args";
+NSString *const EXPLAIN_ROWS = @"rows";
 
-@synthesize storeQueue = _storeQueue;
-@synthesize storeName = _storeName;
-@synthesize user = _user;
-@synthesize isGlobal = _isGlobal;
-@synthesize dbMgr = _dbMgr;
+@implementation SFSmartStore
 
 + (void)initialize
 {
@@ -533,6 +532,26 @@ - (FMResultSet*) executeQueryThrows:(NSString*)sql withDb:(FMDatabase*)db {
 }
 
 - (FMResultSet*) executeQueryThrows:(NSString*)sql withArgumentsInArray:(NSArray*)arguments withDb:(FMDatabase*)db {
+    if (self.captureExplainQueryPlan) {
+        NSString* explainSql = [NSString stringWithFormat:@"EXPLAIN QUERY PLAN %@", sql];
+        NSMutableDictionary* lastPlan = [NSMutableDictionary new];
+        lastPlan[EXPLAIN_SQL] = explainSql;
+        if (arguments.count > 0) lastPlan[EXPLAIN_ARGS] = arguments;
+        NSMutableArray* explainRows = [NSMutableArray new];
+
+        FMResultSet* frs = [db executeQuery:explainSql withArgumentsInArray:arguments];
+        while ([frs next]) {
+            NSMutableDictionary* explainRow = [NSMutableDictionary new];
+            for (NSUInteger i=0; i<frs.columnCount; i++) {
+                explainRow[[frs columnNameForIndex:i]] = [frs stringForColumnIndex:i];
+            }
+            [explainRows addObject:explainRow];
+        }
+        [frs close];
+        lastPlan[EXPLAIN_ROWS] = explainRows;
+        self.lastExplainQueryPlan = lastPlan;
+    }
+
     FMResultSet* result = [db executeQuery:sql withArgumentsInArray:arguments];
     if (!result) {
         [self logAndThrowLastError:[NSString stringWithFormat:@"executeQuery [%@] failed", sql] withDb:db];
@@ -991,10 +1010,15 @@ - (void)registerSoup:(NSString*)soupName withIndexSpecs:(NSArray*)indexSpecs wit
         SFSoupIndex *indexSpec = (SFSoupIndex*) indexSpecs[i];
 
         // for creating the soup table itself in the store db
+        // Column name or expression the db index is on
         NSString *columnName = [NSString stringWithFormat:@"%@_%lu",soupTableName,(unsigned long)i];
-        NSString * columnType = [indexSpec columnType];
-        [createTableStmt appendFormat:@", %@ %@ ",columnName,columnType];
-        [self log:SFLogLevelDebug format:@"adding indexPath: %@ %@  ('%@')",columnName, columnType, [indexSpec path]];
+        if (kValueIndexedWithJSONExtract(indexSpec)) {
+            columnName = [NSString stringWithFormat:@"json_extract(soup, '$.%@')", indexSpec.path];
+        }
+        if (kValueExtractedToColumn(indexSpec)) {
+            NSString * columnType = [indexSpec columnType];
+            [createTableStmt appendFormat:@", %@ %@ ",columnName,columnType];
+        }
 
         // for fts
         if ([indexSpec.indexType isEqualToString:kSoupIndexTypeFullText]) {
@@ -1234,7 +1258,7 @@ - (NSArray *)queryWithQuerySpec:(SFQuerySpec *)querySpec pageIndex:(NSUInteger)p
 
 - (NSArray *)queryWithQuerySpec:(SFQuerySpec *)querySpec pageIndex:(NSUInteger)pageIndex withDb:(FMDatabase*)db
 {
-    [self log:SFLogLevelDebug format:@"queryWithQuerySpec: \nquerySpec:%@ \n", querySpec];
+    // [self log:SFLogLevelDebug format:@"queryWithQuerySpec: \nquerySpec:%@ \n", querySpec];
     NSMutableArray* result = [NSMutableArray arrayWithCapacity:querySpec.pageSize];
 
     // Page
@@ -1245,7 +1269,7 @@ - (NSArray *)queryWithQuerySpec:(SFQuerySpec *)querySpec pageIndex:(NSUInteger)p
     // SQL
     NSString* sql = [self convertSmartSql: querySpec.smartSql withDb:db];
     NSString* limitSql = [@[@"SELECT * FROM (", sql, @") LIMIT ", limit] componentsJoinedByString:@""];
-    [self log:SFLogLevelDebug format:@"queryWithQuerySpec: \nlimitSql:%@ \npageIndex:%d \n", limitSql, pageIndex];
+    // [self log:SFLogLevelDebug format:@"queryWithQuerySpec: \nlimitSql:%@ \npageIndex:%d \n", limitSql, pageIndex];
 
     // Args
     NSArray* args = [querySpec bindsForQuerySpec];
@@ -1342,39 +1366,43 @@ - (NSArray *)retrieveEntries:(NSArray*)soupEntryIds fromSoup:(NSString*)soupName
 - (NSDictionary *)insertOneEntry:(NSDictionary*)entry inSoupTable:(NSString*)soupTableName indices:(NSArray*)indices withDb:(FMDatabase*) db
 {
     NSNumber *nowVal = [self currentTimeInMilliseconds];
-    NSMutableDictionary *values = [NSMutableDictionary dictionaryWithObjectsAndKeys:
-                                        @"", SOUP_COL,
-                                        nowVal, CREATED_COL,
-                                        nowVal, LAST_MODIFIED_COL,
-                                        nil];
-
-    //build up the set of index column values for this new row
-    [self projectIndexedPaths:entry values:values indices:indices typeFilter:nil];
-    [self insertIntoTable:soupTableName values:values withDb:db];
-
-    //set the newly-calculated entry ID so that our next update will update this entry (and not create a new one)
-    NSNumber *newEntryId = [NSNumber numberWithLongLong:[db lastInsertRowId]];
+    NSNumber *newEntryId;
 
+    // Get next id
+    FMResultSet *frs = [self executeQueryThrows:@"SELECT seq FROM SQLITE_SEQUENCE WHERE name = ?" withArgumentsInArray:@[soupTableName] withDb:db];
+    if ([frs next]) {
+        newEntryId = [NSNumber numberWithLongLong:1LL + [frs longLongIntForColumnIndex:0]];
+    }
+    else {
+        // First time, we won't find any rows;
+        newEntryId = [NSNumber numberWithLongLong:1LL];
+    }
+    [frs close];
+
     //clone the entry so that we can insert the new SOUP_ENTRY_ID into the json
     NSMutableDictionary *mutableEntry = [entry mutableCopy];
     [mutableEntry setValue:newEntryId forKey:SOUP_ENTRY_ID];
     [mutableEntry setValue:nowVal forKey:SOUP_LAST_MODIFIED_DATE];
 
     //now update the SOUP_COL (raw json) for the soup entry
     NSString *rawJson = [SFJsonUtils JSONRepresentation:mutableEntry];
-    NSArray *binds = @[rawJson,
-                      newEntryId];
-    NSString *updateSql = [NSString stringWithFormat:@"UPDATE %@ SET %@=? WHERE %@=?", soupTableName, SOUP_COL, ID_COL];
-    //    [self log:SFLogLevelDebug format:@"updateSql: \n %@ \n binds: %@",updateSql,binds];
 
-    [self executeUpdateThrows:updateSql withArgumentsInArray:binds withDb:db];
-
+    NSMutableDictionary *values = [NSMutableDictionary dictionaryWithObjectsAndKeys:
+                                        rawJson, SOUP_COL,
+                                        nowVal, CREATED_COL,
+                                        nowVal, LAST_MODIFIED_COL,
+                                        nil];
+
+    //build up the set of index column values for this new row
+    [self projectIndexedPaths:entry values:values indices:indices typeFilter:kValueExtractedToColumn];
+    [self insertIntoTable:soupTableName values:values withDb:db];
+
     // fts
     if ([SFSoupIndex hasFts:indices]) {
         NSMutableDictionary *ftsValues = [NSMutableDictionary dictionaryWithObjectsAndKeys:
                                           newEntryId, DOCID_COL,
                                           nil];
-        [self projectIndexedPaths:entry values:ftsValues indices:indices typeFilter:kSoupIndexTypeFullText];
+        [self projectIndexedPaths:entry values:ftsValues indices:indices typeFilter:kValueExtractedToFtsColumn];
         [self insertIntoTable:[NSString stringWithFormat:@"%@_fts", soupTableName] values:ftsValues withDb:db];
     }
 
@@ -1395,7 +1423,7 @@ - (NSDictionary *)updateOneEntry:(NSDictionary *)entry
                                     nil];
 
     //build up the set of index column values for this row
-    [self projectIndexedPaths:entry values:values indices:indices typeFilter:nil];
+    [self projectIndexedPaths:entry values:values indices:indices typeFilter:kValueExtractedToColumn];
 
     //clone the entry so that we can modify SOUP_LAST_MODIFIED_DATE
     NSMutableDictionary *mutableEntry = [entry mutableCopy];
@@ -1408,7 +1436,7 @@ - (NSDictionary *)updateOneEntry:(NSDictionary *)entry
     // fts
     if ([SFSoupIndex hasFts:indices]) {
         NSMutableDictionary *ftsValues = [NSMutableDictionary new];
-        [self projectIndexedPaths:entry values:ftsValues indices:indices typeFilter:kSoupIndexTypeFullText];
+        [self projectIndexedPaths:entry values:ftsValues indices:indices typeFilter:kValueExtractedToFtsColumn];
         [self updateTable:[NSString stringWithFormat:@"%@_fts", soupTableName] values:ftsValues entryId:entryId idCol:DOCID_COL withDb:db];
     }
 
@@ -1617,14 +1645,14 @@ - (BOOL) reIndexSoup:(NSString*)soupName withIndexPaths:(NSArray*)indexPaths wit
             NSString *soupElt = [frs stringForColumn:SOUP_COL];
             NSDictionary *entry = [SFJsonUtils objectFromJSONString:soupElt];
             NSMutableDictionary *values = [NSMutableDictionary dictionary];
-            [self projectIndexedPaths:entry values:values indices:indices typeFilter:nil];
+            [self projectIndexedPaths:entry values:values indices:indices typeFilter:kValueExtractedToColumn];
             if ([values count] > 0) {
                 [self updateTable:soupTableName values:values entryId:entryId idCol:ID_COL withDb:db];
             }
             // fts
             if (hasFts) {
                 NSMutableDictionary *ftsValues = [NSMutableDictionary dictionary];
-                [self projectIndexedPaths:entry values:ftsValues indices:indices typeFilter:kSoupIndexTypeFullText];
+                [self projectIndexedPaths:entry values:ftsValues indices:indices typeFilter:kValueExtractedToFtsColumn];
                 if ([ftsValues count] > 0) {
                     [self updateTable:[NSString stringWithFormat:@"%@_fts", soupTableName] values:ftsValues entryId:entryId idCol:DOCID_COL withDb:db];
                 }
@@ -1645,20 +1673,21 @@ - (BOOL) hasFts:(NSString*)soupName withDb:(FMDatabase *)db
 
 #pragma mark - Misc
 
-- (void) projectIndexedPaths:(NSDictionary*)entry values:(NSMutableDictionary*)values indices:(NSArray*)indices typeFilter:(NSString*)typeFilter
+- (void) projectIndexedPaths:(NSDictionary*)entry values:(NSMutableDictionary*)values indices:(NSArray*)indices typeFilter:(SFIndexSpecTypeFilterBlock)typeFilter
 {
     // build up the set of index column values for this row
     for (SFSoupIndex *idx in indices) {
+        if (!typeFilter(idx))
+            continue;
+
         id indexColVal = [SFJsonUtils projectIntoJson:entry path:[idx path]];;
         // values for non-leaf nodes are json-ized
         if ([indexColVal isKindOfClass:[NSDictionary class]] || [indexColVal isKindOfClass:[NSArray class]]) {
             indexColVal = [SFJsonUtils JSONRepresentation:indexColVal options:0];
         }
 
         NSString *colName = [idx columnName];
-        if (typeFilter == nil || [typeFilter isEqualToString:idx.indexType]) {
-            values[colName] = indexColVal != nil ? indexColVal : [NSNull null];
-        }
+        values[colName] = indexColVal != nil ? indexColVal : [NSNull null];
     }
 }
 

libs/SmartStore/SmartStore/Classes/SFSoupIndex.h

@@ -30,6 +30,17 @@ extern NSString * const kSoupIndexTypeString;
 extern NSString * const kSoupIndexTypeInteger;
 extern NSString * const kSoupIndexTypeFloating;
 extern NSString * const kSoupIndexTypeFullText;
+extern NSString * const kSoupIndexTypeJSON1;
+
+
+/**
+ * Index types filter
+ */
+@class SFSoupIndex;
+typedef BOOL (^SFIndexSpecTypeFilterBlock)(SFSoupIndex*);
+extern SFIndexSpecTypeFilterBlock const kValueExtractedToColumn;
+extern SFIndexSpecTypeFilterBlock const kValueExtractedToFtsColumn;
+extern SFIndexSpecTypeFilterBlock const kValueIndexedWithJSONExtract;
 
 /**
  * Definition of an index on a given soup.
@@ -120,3 +131,5 @@ extern NSString * const kSoupIndexTypeFullText;
 - (NSString*) getPathType;
 
 @end
+
+

libs/SmartStore/SmartStore/Classes/SFSoupIndex.m

@@ -28,10 +28,15 @@
 NSString * const kSoupIndexTypeInteger  = @"integer";
 NSString * const kSoupIndexTypeFloating = @"floating";
 NSString * const kSoupIndexTypeFullText = @"full_text";
+NSString * const kSoupIndexTypeJSON1    = @"json1";
 NSString * const kSoupIndexPath         = @"path";
 NSString * const kSoupIndexType         = @"type";
 NSString * const kSoupIndexColumnName   = @"columnName";
 
+SFIndexSpecTypeFilterBlock const kValueExtractedToColumn = ^BOOL (SFSoupIndex* idx) { return ![idx.indexType isEqualToString:kSoupIndexTypeJSON1]; };
+SFIndexSpecTypeFilterBlock const kValueExtractedToFtsColumn = ^BOOL (SFSoupIndex* idx) { return [idx.indexType isEqualToString:kSoupIndexTypeFullText]; };;
+SFIndexSpecTypeFilterBlock const kValueIndexedWithJSONExtract = ^BOOL (SFSoupIndex* idx) { return [idx.indexType isEqualToString:kSoupIndexTypeJSON1]; };;
+
 
 @implementation SFSoupIndex
 
@@ -76,6 +81,8 @@ - (NSString*)columnType {
         result = @"INTEGER";
     } else if ([self.indexType isEqualToString:kSoupIndexTypeFloating]) {
         result = @"REAL";
+    } else if ([self.indexType isEqualToString:kSoupIndexTypeJSON1]) {
+        result = nil;
     }
     return  result;
 }