From c07bd304dbcd2b11e4dd1329a5cfcd1c81ae11f3 Mon Sep 17 00:00:00 2001 From: Yuri Buyanov Date: Wed, 20 Jun 2018 12:24:47 +0300 Subject: [PATCH 1/4] Prevent WAL locking by resetting prepared statements after use Should possibly fix unlimited WAL growing (like in #115) Cached prepared statements in _dbPrepareStmt are not reset until next call, so both automatic and manual checkpoints are failing to occur, causing WAL to grow and never to be merged into main DB file (see http://sqlite.1065341.n5.nabble.com/sqlite3-wal-checkpoint-v2-returning-SQLITE-LOCKED-in-what-circumstances-td86662.html) --- YYCache/YYKVStorage.m | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/YYCache/YYKVStorage.m b/YYCache/YYKVStorage.m index 501dfdd..b4ef22c 100644 --- a/YYCache/YYKVStorage.m +++ b/YYCache/YYKVStorage.m @@ -197,7 +197,14 @@ - (sqlite3_stmt *)_dbPrepareStmt:(NSString *)sql { } CFDictionarySetValue(_dbStmtCache, (__bridge const void *)(sql), stmt); } else { - sqlite3_reset(stmt); + if (sqlite3_stmt_busy(stmt)) { + //just in case someone will forget to sqlite3_reset cached statement + //causing WAL file lock + if (_errorLogsEnabled) { + NSLog(@"%s line:%d WARN: cached statement for query \"%@\" was not reset.", __FUNCTION__, __LINE__, sql); + } + sqlite3_reset(stmt); + } } return stmt; } @@ -239,6 +246,7 @@ - (BOOL)_dbSaveWithKey:(NSString *)key value:(NSData *)value fileName:(NSString sqlite3_bind_blob(stmt, 7, extendedData.bytes, (int)extendedData.length, 0); int result = sqlite3_step(stmt); + sqlite3_reset(stmt); if (result != SQLITE_DONE) { if (_errorLogsEnabled) NSLog(@"%s line:%d sqlite insert error (%d): %s", __FUNCTION__, __LINE__, result, sqlite3_errmsg(_db)); return NO; @@ -253,6 +261,7 @@ - (BOOL)_dbUpdateAccessTimeWithKey:(NSString *)key { sqlite3_bind_int(stmt, 1, (int)time(NULL)); sqlite3_bind_text(stmt, 2, key.UTF8String, -1, NULL); int result = sqlite3_step(stmt); + sqlite3_reset(stmt); if (result != SQLITE_DONE) { if (_errorLogsEnabled) NSLog(@"%s line:%d sqlite update error (%d): %s", __FUNCTION__, __LINE__, result, sqlite3_errmsg(_db)); return NO; @@ -289,6 +298,7 @@ - (BOOL)_dbDeleteItemWithKey:(NSString *)key { sqlite3_bind_text(stmt, 1, key.UTF8String, -1, NULL); int result = sqlite3_step(stmt); + sqlite3_reset(stmt); if (result != SQLITE_DONE) { if (_errorLogsEnabled) NSLog(@"%s line:%d db delete error (%d): %s", __FUNCTION__, __LINE__, result, sqlite3_errmsg(_db)); return NO; @@ -322,6 +332,7 @@ - (BOOL)_dbDeleteItemsWithSizeLargerThan:(int)size { if (!stmt) return NO; sqlite3_bind_int(stmt, 1, size); int result = sqlite3_step(stmt); + sqlite3_reset(stmt); if (result != SQLITE_DONE) { if (_errorLogsEnabled) NSLog(@"%s line:%d sqlite delete error (%d): %s", __FUNCTION__, __LINE__, result, sqlite3_errmsg(_db)); return NO; @@ -335,6 +346,7 @@ - (BOOL)_dbDeleteItemsWithTimeEarlierThan:(int)time { if (!stmt) return NO; sqlite3_bind_int(stmt, 1, time); int result = sqlite3_step(stmt); + sqlite3_reset(stmt); if (result != SQLITE_DONE) { if (_errorLogsEnabled) NSLog(@"%s line:%d sqlite delete error (%d): %s", __FUNCTION__, __LINE__, result, sqlite3_errmsg(_db)); return NO; @@ -380,6 +392,7 @@ - (YYKVStorageItem *)_dbGetItemWithKey:(NSString *)key excludeInlineData:(BOOL)e if (_errorLogsEnabled) NSLog(@"%s line:%d sqlite query error (%d): %s", __FUNCTION__, __LINE__, result, sqlite3_errmsg(_db)); } } + sqlite3_reset(stmt); return item; } @@ -429,11 +442,13 @@ - (NSData *)_dbGetValueWithKey:(NSString *)key { const void *inline_data = sqlite3_column_blob(stmt, 0); int inline_data_bytes = sqlite3_column_bytes(stmt, 0); if (!inline_data || inline_data_bytes <= 0) return nil; + sqlite3_reset(stmt); return [NSData dataWithBytes:inline_data length:inline_data_bytes]; } else { if (result != SQLITE_DONE) { if (_errorLogsEnabled) NSLog(@"%s line:%d sqlite query error (%d): %s", __FUNCTION__, __LINE__, result, sqlite3_errmsg(_db)); } + sqlite3_reset(stmt); return nil; } } @@ -447,6 +462,7 @@ - (NSString *)_dbGetFilenameWithKey:(NSString *)key { if (result == SQLITE_ROW) { char *filename = (char *)sqlite3_column_text(stmt, 0); if (filename && *filename != 0) { + sqlite3_reset(stmt); return [NSString stringWithUTF8String:filename]; } } else { @@ -454,6 +470,7 @@ - (NSString *)_dbGetFilenameWithKey:(NSString *)key { if (_errorLogsEnabled) NSLog(@"%s line:%d sqlite query error (%d): %s", __FUNCTION__, __LINE__, result, sqlite3_errmsg(_db)); } } + sqlite3_reset(stmt); return nil; } @@ -512,6 +529,7 @@ - (NSMutableArray *)_dbGetFilenamesWithSizeLargerThan:(int)size { break; } } while (1); + sqlite3_reset(stmt); return filenames; } @@ -538,6 +556,7 @@ - (NSMutableArray *)_dbGetFilenamesWithTimeEarlierThan:(int)time { break; } } while (1); + sqlite3_reset(stmt); return filenames; } @@ -570,6 +589,7 @@ - (NSMutableArray *)_dbGetItemSizeInfoOrderByTimeAscWithLimit:(int)count { break; } } while (1); + sqlite3_reset(stmt); return items; } @@ -581,9 +601,12 @@ - (int)_dbGetItemCountWithKey:(NSString *)key { int result = sqlite3_step(stmt); if (result != SQLITE_ROW) { if (_errorLogsEnabled) NSLog(@"%s line:%d sqlite query error (%d): %s", __FUNCTION__, __LINE__, result, sqlite3_errmsg(_db)); + sqlite3_reset(stmt); return -1; } - return sqlite3_column_int(stmt, 0); + int count = sqlite3_column_int(stmt, 0); + sqlite3_reset(stmt); + return count; } - (int)_dbGetTotalItemSize { @@ -593,9 +616,12 @@ - (int)_dbGetTotalItemSize { int result = sqlite3_step(stmt); if (result != SQLITE_ROW) { if (_errorLogsEnabled) NSLog(@"%s line:%d sqlite query error (%d): %s", __FUNCTION__, __LINE__, result, sqlite3_errmsg(_db)); + sqlite3_reset(stmt); return -1; } - return sqlite3_column_int(stmt, 0); + int size = sqlite3_column_int(stmt, 0); + sqlite3_reset(stmt); + return size; } - (int)_dbGetTotalItemCount { @@ -607,7 +633,9 @@ - (int)_dbGetTotalItemCount { if (_errorLogsEnabled) NSLog(@"%s line:%d sqlite query error (%d): %s", __FUNCTION__, __LINE__, result, sqlite3_errmsg(_db)); return -1; } - return sqlite3_column_int(stmt, 0); + int count = sqlite3_column_int(stmt, 0); + sqlite3_reset(stmt); + return count; } From 3fee8fc1cd9bc52889c9c491bef6d61c25c5e29f Mon Sep 17 00:00:00 2001 From: Yuri Buyanov Date: Thu, 21 Jun 2018 18:40:17 +0300 Subject: [PATCH 2/4] added sqlite3_wal_checkpoint non-ok result logging --- YYCache/YYKVStorage.m | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/YYCache/YYKVStorage.m b/YYCache/YYKVStorage.m index b4ef22c..844a075 100644 --- a/YYCache/YYKVStorage.m +++ b/YYCache/YYKVStorage.m @@ -169,7 +169,10 @@ - (BOOL)_dbInitialize { - (void)_dbCheckpoint { if (![self _dbCheck]) return; // Cause a checkpoint to occur, merge `sqlite-wal` file to `sqlite` file. - sqlite3_wal_checkpoint(_db, NULL); + int result = sqlite3_wal_checkpoint(_db, NULL); + if (result != SQLITE_OK && _errorLogsEnabled) { + NSLog(@"%s line:%d sqlite WAL checkpoint error (%d)", __FUNCTION__, __LINE__, result); + } } - (BOOL)_dbExecute:(NSString *)sql { From 1e13238b2e46b4aecc7f3cfec557dfc37730eec0 Mon Sep 17 00:00:00 2001 From: Yuri Buyanov Date: Mon, 20 Jan 2020 18:25:16 +0300 Subject: [PATCH 3/4] sqlite3_reset before inline_data check in _dbGetValueWithKey https://github.com/ibireme/YYCache/pull/129/files/c07bd304dbcd2b11e4dd1329a5cfcd1c81ae11f3#r356394235 --- YYCache/YYKVStorage.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/YYCache/YYKVStorage.m b/YYCache/YYKVStorage.m index 844a075..89ed7bb 100644 --- a/YYCache/YYKVStorage.m +++ b/YYCache/YYKVStorage.m @@ -444,8 +444,8 @@ - (NSData *)_dbGetValueWithKey:(NSString *)key { if (result == SQLITE_ROW) { const void *inline_data = sqlite3_column_blob(stmt, 0); int inline_data_bytes = sqlite3_column_bytes(stmt, 0); - if (!inline_data || inline_data_bytes <= 0) return nil; sqlite3_reset(stmt); + if (!inline_data || inline_data_bytes <= 0) return nil; return [NSData dataWithBytes:inline_data length:inline_data_bytes]; } else { if (result != SQLITE_DONE) { From 17b318700e31382209532ac90d8cb25769cb7fdb Mon Sep 17 00:00:00 2001 From: Yuri Buyanov Date: Tue, 21 Jan 2020 11:17:15 +0300 Subject: [PATCH 4/4] -[YYMemoryCache objectForKey:] unlock mutex after getting the result value --- YYCache/YYMemoryCache.m | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/YYCache/YYMemoryCache.m b/YYCache/YYMemoryCache.m index 9042d9b..0a97484 100644 --- a/YYCache/YYMemoryCache.m +++ b/YYCache/YYMemoryCache.m @@ -402,8 +402,9 @@ - (id)objectForKey:(id)key { node->_time = CACurrentMediaTime(); [_lru bringNodeToHead:node]; } + id result = node ? node->_value : nil; pthread_mutex_unlock(&_lock); - return node ? node->_value : nil; + return result; } - (void)setObject:(id)object forKey:(id)key {