Merge pull request #448 from Aiven-Open/ivanyu/demo-disk-cache

Refresh demo
Aiven-Open · Nov 16, 2023 · ef81d0a · ef81d0a
2 parents 47928ab + bf1f5c0
commit ef81d0a
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 18 deletions.
diff --git a/demo/Makefile b/demo/Makefile
@@ -22,9 +22,9 @@ bootstrap_server_host=localhost:9092
 topic=topic1
 partitions=1
 replication_factor=1
-segment=1000000 # 1MB
-retention_bytes=20000000 # 20MB
-local_retention_bytes=5000000 # 5MB
+segment=1048576 # 1 MiB
+retention_bytes=104857600 # 100 MiB
+local_retention_bytes=1
 retention_ms=360000000 # 100 hours
 local_retention_ms=1000 # 1 second
 

diff --git a/demo/compose-azure-blob-azurite.yml b/demo/compose-azure-blob-azurite.yml
@@ -59,9 +59,11 @@ services:
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*:/tiered-storage-for-apache-kafka/azure/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
-      KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.InMemoryChunkCache"
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: -1
+      KAFKA_RSM_CONFIG_CHUNK_SIZE: 4194304 # 4 MiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.DiskBasedChunkCache"
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PATH: /home/appuser/kafka-tiered-storage-cache
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: 1073741824 # 1 GiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PREFETCH_MAX_SIZE: 16777216 # 16 MiB
       KAFKA_RSM_CONFIG_CUSTOM_METADATA_FIELDS_INCLUDE: "REMOTE_SIZE"
       # Storage backend
       KAFKA_RSM_CONFIG_KEY_PREFIX: "tiered-storage-demo/"

diff --git a/demo/compose-gcs-fake-gcs-server.yml b/demo/compose-gcs-fake-gcs-server.yml
@@ -59,9 +59,11 @@ services:
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*:/tiered-storage-for-apache-kafka/gcs/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
-      KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.InMemoryChunkCache"
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: -1
+      KAFKA_RSM_CONFIG_CHUNK_SIZE: 4194304 # 4 MiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.DiskBasedChunkCache"
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PATH: /home/appuser/kafka-tiered-storage-cache
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: 1073741824 # 1 GiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PREFETCH_MAX_SIZE: 16777216 # 16 MiB
       KAFKA_RSM_CONFIG_CUSTOM_METADATA_FIELDS_INCLUDE: "REMOTE_SIZE"
       # Storage backend
       KAFKA_RSM_CONFIG_KEY_PREFIX: "tiered-storage-demo/"

diff --git a/demo/compose-local-fs.yml b/demo/compose-local-fs.yml
@@ -58,9 +58,11 @@ services:
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
-      KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.InMemoryChunkCache"
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: -1
+      KAFKA_RSM_CONFIG_CHUNK_SIZE: 4194304 # 4 MiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.DiskBasedChunkCache"
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PATH: /home/appuser/kafka-tiered-storage-cache
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: 1073741824 # 1 GiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PREFETCH_MAX_SIZE: 16777216 # 16 MiB
       KAFKA_RSM_CONFIG_CUSTOM_METADATA_FIELDS_INCLUDE: "REMOTE_SIZE"
       # Storage backend
       KAFKA_RSM_CONFIG_KEY_PREFIX: "tiered-storage-demo/"

diff --git a/demo/compose-s3-aws.yml b/demo/compose-s3-aws.yml
@@ -58,9 +58,11 @@ services:
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*:/tiered-storage-for-apache-kafka/s3/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
-      KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.InMemoryChunkCache"
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: -1
+      KAFKA_RSM_CONFIG_CHUNK_SIZE: 4194304 # 4 MiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.DiskBasedChunkCache"
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PATH: /home/appuser/kafka-tiered-storage-cache
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: 1073741824 # 1 GiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PREFETCH_MAX_SIZE: 16777216 # 16 MiB
       KAFKA_RSM_CONFIG_CUSTOM_METADATA_FIELDS_INCLUDE: "REMOTE_SIZE"
       # Storage backend
       KAFKA_RSM_CONFIG_KEY_PREFIX: "tiered-storage-demo/"

diff --git a/demo/compose-s3-minio.yml b/demo/compose-s3-minio.yml
@@ -59,9 +59,11 @@ services:
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*:/tiered-storage-for-apache-kafka/s3/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
-      KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.InMemoryChunkCache"
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: -1
+      KAFKA_RSM_CONFIG_CHUNK_SIZE: 4194304 # 4 MiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.DiskBasedChunkCache"
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PATH: /home/appuser/kafka-tiered-storage-cache
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: 1073741824 # 1 GiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PREFETCH_MAX_SIZE: 16777216 # 16 MiB
       KAFKA_RSM_CONFIG_CUSTOM_METADATA_FIELDS_INCLUDE: "REMOTE_SIZE"
       # Storage backend
       KAFKA_RSM_CONFIG_KEY_PREFIX: "tiered-storage-demo/"