forked from seedvault-app/seedvault
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
5 changed files
with
166 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
147 changes: 147 additions & 0 deletions
147
storage/lib/src/main/java/org/calyxos/backup/storage/check/Checker.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| /* | ||
| * SPDX-FileCopyrightText: 2024 The Calyx Institute | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.calyxos.backup.storage.check | ||
|
|
||
| import android.util.Log | ||
| import com.google.protobuf.InvalidProtocolBufferException | ||
| import org.calyxos.backup.storage.SnapshotRetriever | ||
| import org.calyxos.backup.storage.api.BackupObserver | ||
| import org.calyxos.backup.storage.api.StoredSnapshot | ||
| import org.calyxos.backup.storage.backup.Backup.Companion.VERSION | ||
| import org.calyxos.backup.storage.backup.ChunksCacheRepopulater | ||
| import org.calyxos.backup.storage.crypto.ChunkCrypto | ||
| import org.calyxos.backup.storage.crypto.StreamCrypto | ||
| import org.calyxos.backup.storage.db.Db | ||
| import org.calyxos.backup.storage.restore.readVersion | ||
| import org.calyxos.seedvault.core.backends.Backend | ||
| import org.calyxos.seedvault.core.backends.FileBackupFileType | ||
| import org.calyxos.seedvault.core.backends.TopLevelFolder | ||
| import org.calyxos.seedvault.core.crypto.KeyManager | ||
| import org.calyxos.seedvault.core.toHexString | ||
| import java.io.IOException | ||
| import java.security.GeneralSecurityException | ||
| import kotlin.math.roundToLong | ||
|
|
||
| private val TAG = Checker::class.simpleName | ||
|
|
||
| internal class Checker( | ||
| private val db: Db, | ||
| private val backendGetter: () -> Backend, | ||
| private val snapshotRetriever: SnapshotRetriever, | ||
| private val keyManager: KeyManager, | ||
| private val cacheRepopulater: ChunksCacheRepopulater, | ||
| private val androidId: String, | ||
| private val streamCrypto: StreamCrypto = StreamCrypto, | ||
| private val chunkCrypto: ChunkCrypto = ChunkCrypto, | ||
| ) { | ||
|
|
||
| private val backend get() = backendGetter() | ||
|
|
||
| private val streamKey | ||
| get() = try { | ||
| streamCrypto.deriveStreamKey(keyManager.getMainKey()) | ||
| } catch (e: GeneralSecurityException) { | ||
| throw AssertionError(e) | ||
| } | ||
| private val mac | ||
| get() = try { | ||
| chunkCrypto.getMac(ChunkCrypto.deriveChunkIdKey(keyManager.getMainKey())) | ||
| } catch (e: GeneralSecurityException) { | ||
| throw AssertionError(e) | ||
| } | ||
|
|
||
| fun getBackupSize(): Long { | ||
| return db.getChunksCache().getSizeOfCachedChunks() | ||
| } | ||
|
|
||
| @Throws( | ||
| IOException::class, | ||
| GeneralSecurityException::class, | ||
| InvalidProtocolBufferException::class, | ||
| ) | ||
| suspend fun check(percent: Int, backupObserver: BackupObserver?) { | ||
| check(percent in 0..100) { "Invalid percentage: $percent" } | ||
|
|
||
| // get all snapshots and blobs on storage | ||
| val topLevelFolder = TopLevelFolder("$androidId.sv") | ||
| val storedSnapshots = mutableListOf<StoredSnapshot>() | ||
| val availableChunkIds = mutableMapOf<String, Long>() | ||
| backend.list( | ||
| topLevelFolder, | ||
| FileBackupFileType.Snapshot::class, | ||
| FileBackupFileType.Blob::class, | ||
| ) { fileInfo -> | ||
| when (fileInfo.fileHandle) { | ||
| is FileBackupFileType.Snapshot -> { | ||
| val handle = fileInfo.fileHandle as FileBackupFileType.Snapshot | ||
| val storedSnapshot = StoredSnapshot(handle.topLevelFolder.name, handle.time) | ||
| storedSnapshots.add(storedSnapshot) | ||
| } | ||
| is FileBackupFileType.Blob -> | ||
| availableChunkIds[fileInfo.fileHandle.name] = fileInfo.size | ||
| else -> error("Unexpected FileHandle: $fileInfo") | ||
| } | ||
| } | ||
| // ensure our local ChunksCache is up to date | ||
| if (!db.getChunksCache().areAllAvailableChunksCached(db, availableChunkIds.keys)) { | ||
| Log.i(TAG, "Not all available chunks cached, rebuild local cache...") | ||
| cacheRepopulater.repopulate(streamKey, availableChunkIds) | ||
| } | ||
| // parse snapshots | ||
| val snapshots = storedSnapshots.map { snapshotRetriever.getSnapshot(streamKey, it) } | ||
| Log.i(TAG, "Found ${storedSnapshots.size} snapshots, ${snapshots.size} readable.") | ||
| // get all referenced chunkIds | ||
| val referencedChunkIds = mutableSetOf<String>() | ||
| snapshots.forEach { snapshot -> | ||
| snapshot.mediaFilesList.forEach { referencedChunkIds.addAll(it.chunkIdsList) } | ||
| snapshot.documentFilesList.forEach { referencedChunkIds.addAll(it.chunkIdsList) } | ||
| } | ||
| // calculate chunks that are missing | ||
| val missingChunkIds = referencedChunkIds - availableChunkIds.keys | ||
| Log.i( | ||
| TAG, "Found ${referencedChunkIds.size} referenced chunks, " + | ||
| "${missingChunkIds.size} missing." | ||
| ) | ||
|
|
||
| val chunkIdMac = mac // keep a copy of the mac | ||
| checkBlobSample(referencedChunkIds, percent).forEach { chunkId -> | ||
| val handle = FileBackupFileType.Blob(androidId, chunkId) | ||
| val cachedChunk = db.getChunksCache().get(chunkId) | ||
| // if chunk is not in DB, it isn't available on backend, | ||
| // so missing version doesn't matter | ||
| val version = cachedChunk?.version ?: VERSION | ||
| val readId = backend.load(handle).use { inputStream -> | ||
| inputStream.readVersion(version.toInt()) | ||
| val ad = streamCrypto.getAssociatedDataForChunk(chunkId, version) | ||
| streamCrypto.newDecryptingStream(streamKey, inputStream, ad) | ||
| .use { decryptedStream -> | ||
| chunkIdMac.doFinal(decryptedStream.readAllBytes()).toHexString() | ||
| } | ||
| } | ||
| if (readId != chunkId) { | ||
| Log.w(TAG, "Wrong chunkId $readId for $chunkId") | ||
| } else { | ||
| Log.i(TAG, "Checked chunkId $chunkId") | ||
| } | ||
| } | ||
|
|
||
| } | ||
|
|
||
| private fun checkBlobSample(referencedChunkIds: Set<String>, percent: Int): List<String> { | ||
| val size = getBackupSize() | ||
| val targetSize = (size * (percent.toDouble() / 100)).roundToLong() | ||
| val blobSample = mutableListOf<String>() | ||
| val iterator = referencedChunkIds.shuffled().iterator() | ||
| var currentSize = 0L | ||
| while (currentSize < targetSize && iterator.hasNext()) { | ||
| val chunkId = iterator.next() | ||
| blobSample.add(chunkId) | ||
| // we ensure cache consistency above, so chunks not in cache don't exist anymore | ||
| currentSize += db.getChunksCache().get(chunkId)?.size ?: 0L | ||
| } | ||
| return blobSample | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters