From 1ff2526577c17063ce2a4fdfebdde9ebe3da30e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C9=91rry=20Shiv=C9=91m?= Date: Thu, 26 Sep 2024 12:43:20 +0530 Subject: [PATCH] perf: Migrate epub caching system from JSON to Protocol Buffers (#225) Fixes OOM errors in some cases and enhances the efficiency of caching mechanism, significantly reducing serialization/deserialization times, memory footprint as well as size of cached data on disk. Signed-off-by: starry-shivam --- app/build.gradle | 3 +- .../java/com/starry/myne/epub/EpubParser.kt | 1 + .../myne/epub/{ => cache}/BitmapSerializer.kt | 26 +++++----- .../starry/myne/epub/{ => cache}/EpubCache.kt | 51 ++++++++++++------- .../starry/myne/epub/cache/epub_cache.proto | 36 +++++++++++++ .../com/starry/myne/epub/models/EpubBook.kt | 21 ++++---- .../starry/myne/epub/models/EpubChapter.kt | 12 +++-- .../com/starry/myne/epub/models/EpubImage.kt | 7 ++- 8 files changed, 111 insertions(+), 46 deletions(-) rename app/src/main/java/com/starry/myne/epub/{ => cache}/BitmapSerializer.kt (68%) rename app/src/main/java/com/starry/myne/epub/{ => cache}/EpubCache.kt (71%) create mode 100644 app/src/main/java/com/starry/myne/epub/cache/epub_cache.proto diff --git a/app/build.gradle b/app/build.gradle index 67d534b..d258671 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -117,7 +117,8 @@ dependencies { // Android 12+ splash API. implementation 'androidx.core:core-splashscreen:1.0.1' // KotlinX Serialization library. - implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3" + implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.7.3" + implementation "org.jetbrains.kotlinx:kotlinx-serialization-protobuf:1.7.3" // OkHttp library. implementation "com.squareup.okhttp3:okhttp:4.12.0" implementation "com.squareup.okhttp3:logging-interceptor:4.12.0" diff --git a/app/src/main/java/com/starry/myne/epub/EpubParser.kt b/app/src/main/java/com/starry/myne/epub/EpubParser.kt index 1e741b8..59891e1 100644 --- a/app/src/main/java/com/starry/myne/epub/EpubParser.kt +++ b/app/src/main/java/com/starry/myne/epub/EpubParser.kt @@ -21,6 +21,7 @@ import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory import android.util.Log +import com.starry.myne.epub.cache.EpubCache import com.starry.myne.epub.models.EpubBook import com.starry.myne.epub.models.EpubChapter import com.starry.myne.epub.models.EpubImage diff --git a/app/src/main/java/com/starry/myne/epub/BitmapSerializer.kt b/app/src/main/java/com/starry/myne/epub/cache/BitmapSerializer.kt similarity index 68% rename from app/src/main/java/com/starry/myne/epub/BitmapSerializer.kt rename to app/src/main/java/com/starry/myne/epub/cache/BitmapSerializer.kt index 54e488a..ca63175 100644 --- a/app/src/main/java/com/starry/myne/epub/BitmapSerializer.kt +++ b/app/src/main/java/com/starry/myne/epub/cache/BitmapSerializer.kt @@ -14,36 +14,38 @@ * limitations under the License. */ -package com.starry.myne.epub +package com.starry.myne.epub.cache import android.graphics.Bitmap import android.graphics.BitmapFactory import kotlinx.serialization.KSerializer -import kotlinx.serialization.descriptors.PrimitiveKind -import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.builtins.ByteArraySerializer import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.descriptors.buildClassSerialDescriptor +import kotlinx.serialization.descriptors.element import kotlinx.serialization.encoding.Decoder import kotlinx.serialization.encoding.Encoder import java.io.ByteArrayOutputStream +/** + * A [KSerializer] for [Bitmap] objects. + * It serializes the bitmap to a byte array and deserializes it back to a bitmap. + */ object BitmapSerializer : KSerializer { - override val descriptor: SerialDescriptor = - PrimitiveSerialDescriptor("Bitmap", PrimitiveKind.STRING) + override val descriptor: SerialDescriptor = buildClassSerialDescriptor("Bitmap") { + element("bytes") + } override fun serialize(encoder: Encoder, value: Bitmap) { val stream = ByteArrayOutputStream() value.compress(Bitmap.CompressFormat.PNG, 100, stream) val byteArray = stream.toByteArray() - encoder.encodeString( - android.util.Base64.encodeToString( - byteArray, android.util.Base64.DEFAULT - ) - ) + encoder.encodeSerializableValue(ByteArraySerializer(), byteArray) } override fun deserialize(decoder: Decoder): Bitmap { - val byteArray = - android.util.Base64.decode(decoder.decodeString(), android.util.Base64.DEFAULT) + val byteArray = decoder.decodeSerializableValue(ByteArraySerializer()) return BitmapFactory.decodeByteArray(byteArray, 0, byteArray.size) } } + diff --git a/app/src/main/java/com/starry/myne/epub/EpubCache.kt b/app/src/main/java/com/starry/myne/epub/cache/EpubCache.kt similarity index 71% rename from app/src/main/java/com/starry/myne/epub/EpubCache.kt rename to app/src/main/java/com/starry/myne/epub/cache/EpubCache.kt index 076cb4c..498d9ed 100644 --- a/app/src/main/java/com/starry/myne/epub/EpubCache.kt +++ b/app/src/main/java/com/starry/myne/epub/cache/EpubCache.kt @@ -14,19 +14,21 @@ * limitations under the License. */ -package com.starry.myne.epub +package com.starry.myne.epub.cache import android.content.Context import android.util.Log import com.starry.myne.epub.models.EpubBook -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.protobuf.ProtoBuf +import kotlinx.serialization.protobuf.schema.ProtoBufSchemaGenerator import java.io.File /** - * A cache for storing epub books. + * A cache storage based on Protocol Buffers for storing [EpubBook] objects. + * The cache is stored in the app's cache directory. * - * @param context The context. + * @param context The context of the application. */ class EpubCache(private val context: Context) { @@ -36,7 +38,7 @@ class EpubCache(private val context: Context) { private const val CACHE_VERSION_FILE = "cache_version" // Increment this if the cache format changes. - private const val EPUB_CACHE_VERSION = 1 + private const val EPUB_CACHE_VERSION = 2 } init { @@ -44,6 +46,9 @@ class EpubCache(private val context: Context) { checkCacheVersion() } + @OptIn(ExperimentalSerializationApi::class) + private val protobuf = ProtoBuf { encodeDefaults = true } + private fun getPath(): String { return context.cacheDir.absolutePath + File.separator + EPUB_CACHE } @@ -52,6 +57,7 @@ class EpubCache(private val context: Context) { return getPath() + File.separator + CACHE_VERSION_FILE } + // Checks if the cache version matches the current version. private fun checkCacheVersion() { Log.d(TAG, "Checking cache version") val versionFile = File(getVersionFilePath()) @@ -69,6 +75,7 @@ class EpubCache(private val context: Context) { } } + // Saves the current cache version. private fun saveCacheVersion() { Log.d(TAG, "Saving cache version") val versionFile = File(getVersionFilePath()) @@ -91,35 +98,45 @@ class EpubCache(private val context: Context) { } } + // Used for debugging purposes. + @Suppress("unused") + @OptIn(ExperimentalSerializationApi::class) + private fun printSchema() { + val protoSchema = + ProtoBufSchemaGenerator.generateSchemaText(EpubBook.serializer().descriptor) + Log.d(TAG, "Proto schema: $protoSchema") + } + /** - * Adds a book to the cache. + * Inserts a book into the cache. * - * @param book The book to add. + * @param book The book to insert. * @param filepath The path to the book file. */ + @OptIn(ExperimentalSerializationApi::class) fun put(book: EpubBook, filepath: String) { Log.d(TAG, "Inserting book into cache: ${book.title}") val fileName = File(filepath).nameWithoutExtension - val bookFile = File(getPath(), "$fileName.json") - val jsonString = Json.encodeToString(book) - bookFile.writeText(jsonString) + val bookFile = File(getPath(), "$fileName.protobuf") + val protoBytes = protobuf.encodeToByteArray(EpubBook.serializer(), book) + bookFile.writeBytes(protoBytes) } /** * Gets a book from the cache. - * If the book is not cached, null is returned. * * @param filepath The path to the book file. * @return The book if it is cached, null otherwise. */ + @OptIn(ExperimentalSerializationApi::class) fun get(filepath: String): EpubBook? { Log.d(TAG, "Getting book from cache: $filepath") val fileName = File(filepath).nameWithoutExtension - val bookFile = File(getPath(), "$fileName.json") + val bookFile = File(getPath(), "$fileName.protobuf") return if (bookFile.exists()) { Log.d(TAG, "Book found in cache: $filepath") - val jsonString = bookFile.readText() - Json.decodeFromString(jsonString) + val protoBytes = bookFile.readBytes() + protobuf.decodeFromByteArray(EpubBook.serializer(), protoBytes) } else { Log.d(TAG, "Book not found in cache: $filepath") null @@ -134,7 +151,7 @@ class EpubCache(private val context: Context) { fun remove(filepath: String): Boolean { Log.d(TAG, "Removing book from cache: $filepath") val fileName = File(filepath).nameWithoutExtension - val bookFile = File(getPath(), "$fileName.json") + val bookFile = File(getPath(), "$fileName.protobuf") return if (bookFile.exists()) { bookFile.delete() } else { @@ -150,7 +167,7 @@ class EpubCache(private val context: Context) { */ fun isCached(filepath: String): Boolean { val fileName = File(filepath).nameWithoutExtension - val bookFile = File(getPath(), "$fileName.json") + val bookFile = File(getPath(), "$fileName.protobuf") return bookFile.exists() } } diff --git a/app/src/main/java/com/starry/myne/epub/cache/epub_cache.proto b/app/src/main/java/com/starry/myne/epub/cache/epub_cache.proto new file mode 100644 index 0000000..a4174f4 --- /dev/null +++ b/app/src/main/java/com/starry/myne/epub/cache/epub_cache.proto @@ -0,0 +1,36 @@ +// Generated schema file for 'com.starry.myne.epub.models.EpubBook' +// For reference only, not actually used to generate protobuf code + +syntax = "proto2"; + +// serial name 'com.starry.myne.epub.models.EpubBook' +message EpubBook { + required string fileName = 1; + required string title = 2; + required string author = 3; + required string language = 4; + optional Bitmap coverImage = 5; + // WARNING: a default value decoded when value is missing + repeated EpubChapter chapters = 6; + // WARNING: a default value decoded when value is missing + repeated EpubImage images = 7; +} + +// serial name 'Bitmap?' +message Bitmap { + required bytes bytes = 1; +} + +// serial name 'com.starry.myne.epub.models.EpubChapter' +message EpubChapter { + required string chapterId = 1; + required string absPath = 2; + required string title = 3; + required string body = 4; +} + +// serial name 'com.starry.myne.epub.models.EpubImage' +message EpubImage { + required string absPath = 1; + required bytes image = 2; +} \ No newline at end of file diff --git a/app/src/main/java/com/starry/myne/epub/models/EpubBook.kt b/app/src/main/java/com/starry/myne/epub/models/EpubBook.kt index 8eaa239..7cd93a9 100644 --- a/app/src/main/java/com/starry/myne/epub/models/EpubBook.kt +++ b/app/src/main/java/com/starry/myne/epub/models/EpubBook.kt @@ -18,8 +18,10 @@ package com.starry.myne.epub.models import android.graphics.Bitmap -import com.starry.myne.epub.BitmapSerializer +import com.starry.myne.epub.cache.BitmapSerializer +import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.Serializable +import kotlinx.serialization.protobuf.ProtoNumber /** * Represents an epub book. @@ -33,13 +35,12 @@ import kotlinx.serialization.Serializable * @param images The list of images in the book. */ @Serializable -data class EpubBook( - val fileName: String, - val title: String, - val author: String, - val language: String, - @Serializable(with = BitmapSerializer::class) - val coverImage: Bitmap?, - val chapters: List, - val images: List +data class EpubBook @OptIn(ExperimentalSerializationApi::class) constructor( + @ProtoNumber(1) val fileName: String, + @ProtoNumber(2) val title: String, + @ProtoNumber(3) val author: String, + @ProtoNumber(4) val language: String, + @ProtoNumber(5) @Serializable(with = BitmapSerializer::class) val coverImage: Bitmap?, + @ProtoNumber(6) val chapters: List = emptyList(), + @ProtoNumber(7) val images: List = emptyList() ) \ No newline at end of file diff --git a/app/src/main/java/com/starry/myne/epub/models/EpubChapter.kt b/app/src/main/java/com/starry/myne/epub/models/EpubChapter.kt index 579a941..a916b33 100644 --- a/app/src/main/java/com/starry/myne/epub/models/EpubChapter.kt +++ b/app/src/main/java/com/starry/myne/epub/models/EpubChapter.kt @@ -17,7 +17,9 @@ package com.starry.myne.epub.models +import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.Serializable +import kotlinx.serialization.protobuf.ProtoNumber /** * Represents a chapter in an epub book. @@ -27,9 +29,9 @@ import kotlinx.serialization.Serializable * @param body The body of the chapter. */ @Serializable -data class EpubChapter( - val chapterId: String, - val absPath: String, - val title: String, - val body: String +data class EpubChapter @OptIn(ExperimentalSerializationApi::class) constructor( + @ProtoNumber(1) val chapterId: String, + @ProtoNumber(2) val absPath: String, + @ProtoNumber(3) val title: String, + @ProtoNumber(4) val body: String ) \ No newline at end of file diff --git a/app/src/main/java/com/starry/myne/epub/models/EpubImage.kt b/app/src/main/java/com/starry/myne/epub/models/EpubImage.kt index b16015e..cfdb7e9 100644 --- a/app/src/main/java/com/starry/myne/epub/models/EpubImage.kt +++ b/app/src/main/java/com/starry/myne/epub/models/EpubImage.kt @@ -17,7 +17,9 @@ package com.starry.myne.epub.models +import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.Serializable +import kotlinx.serialization.protobuf.ProtoNumber /** * Represents an image in an epub book. @@ -26,7 +28,10 @@ import kotlinx.serialization.Serializable * @param image The image data. */ @Serializable -data class EpubImage(val absPath: String, val image: ByteArray) { +data class EpubImage @OptIn(ExperimentalSerializationApi::class) constructor( + @ProtoNumber(1) val absPath: String, + @ProtoNumber(2) val image: ByteArray +) { override fun equals(other: Any?): Boolean { if (this === other) return true if (javaClass != other?.javaClass) return false