Merge pull request #31 from crowdproj/wip-3

deploy (docker-compose-app.yml)
crowdproj · Apr 11, 2024 · 42a66c0 · 42a66c0
2 parents c4907e0 + 225507e
commit 42a66c0
Show file tree

Hide file tree

Showing 36 changed files with 401 additions and 166 deletions.
diff --git a/README.md b/README.md
@@ -30,8 +30,10 @@ This is a second edition, the first one is a java-spring application, which home
 
 #### Build and run:
 
-- `$ docker pull sszuev/open-tutor-standalone:latest && docker run --name open-tutor-standalone-app -p 8080:8080 sszuev/open-tutor-standalone:latest`
-- [app-ktor/README](./app-ktor/README.md)
+- standalone
+  version `$ docker pull sszuev/open-tutor-standalone:latest && docker run --name open-tutor-standalone-app -p 8080:8080 sszuev/open-tutor-standalone:latest`
+- prod version [tutor-deploy/README](./tutor-deploy/README.md)
+- for more info see [app-ktor/README](./app-ktor/README.md)
 
 #### License:
 - Apache License Version 2.0
diff --git a/app-ktor/README.md b/app-ktor/README.md
@@ -20,11 +20,27 @@ $ docker rm -v open-tutor-app
 $ docker run --name open-tutor-app -p 8080:8080 sszuev/open-tutor-standalone:latest
 ```
 
-#### build and run application using docker & gradle:
+#### build and run standalone application using docker & gradle:
 ```shell
 $ docker rm -v open-tutor-app
+$ docker rmi sszuev/open-tutor-standalone:latest
 $ gradle clean build dockerBuildImage -Dstandalone=true
 $ docker run --name open-tutor-app -p 8080:8080 sszuev/open-tutor-standalone:latest  
 ```
 
+#### build and run prod application using docker & gradle:
+
+```shell
+$ docker rm tutor-deploy-flashcards-tts-server-1
+$ docker rm tutor-deploy-flashcards-app-1
+$ docker rmi sszuev/open-tutor-tts-server:2.0.0-snapshot
+$ docker rmi sszuev/open-tutor:2.0.0-snapshot
+$ cd ../tts-server
+$ gradle clean build dockerBuildImage
+$ cd ../app-ktor
+$ gradle clean build dockerBuildImage
+$ cd ../tutor-deploy
+$ docker-compose -f docker-compose-app.yml up  
+```
+
 #### After build and run, the application will be available via http://localhost:8080
diff --git a/app-ktor/build.gradle.kts b/app-ktor/build.gradle.kts
@@ -76,7 +76,7 @@ tasks.test {
 }
 
 application {
-    mainClass.set("com.gitlab.sszuev.flashcards.MainKt")
+    mainClass.set("com.gitlab.sszuev.flashcards.AppMainKt")
 }
 
 tasks.create("createTagFile") {
@@ -89,15 +89,32 @@ tasks.create("createTagFile") {
 }
 
 tasks.dockerCreateDockerfile {
-    copyFile("./resources/data/users.csv", "/app/userdata/users.csv")
-    copyFile("./resources/data/dictionaries.csv", "/app/userdata/dictionaries.csv")
-    copyFile("./resources/data/cards.csv", "/app/userdata/cards.csv")
+    if (System.getProperty("standalone") != null) {
+        copyFile("./resources/data/users.csv", "/app/userdata/users.csv")
+        copyFile("./resources/data/dictionaries.csv", "/app/userdata/dictionaries.csv")
+        copyFile("./resources/data/cards.csv", "/app/userdata/cards.csv")
+    } else {
+        arg("TTS_CLIENT_RABBITMQ_HOST")
+        arg("KEYCLOAK_AUTHORIZE_ADDRESS")
+        arg("KEYCLOAK_ACCESS_TOKEN_ADDRESS")
+        arg("DB_PG_URL")
+        arg("DB_PG_USER")
+        arg("DB_PG_PWD")
+        arg("PORT")
+        environmentVariable("TTS_CLIENT_RABBITMQ_HOST", "\${TTS_CLIENT_RABBITMQ_HOST}")
+        environmentVariable("KEYCLOAK_AUTHORIZE_ADDRESS", "\${KEYCLOAK_AUTHORIZE_ADDRESS}")
+        environmentVariable("KEYCLOAK_ACCESS_TOKEN_ADDRESS", "\${KEYCLOAK_ACCESS_TOKEN_ADDRESS}")
+        environmentVariable("DB_PG_URL", "\${DB_PG_URL}")
+        environmentVariable("DB_PG_USER", "\${DB_PG_USER}")
+        environmentVariable("DB_PG_PWD", "\${DB_PG_PWD}")
+        environmentVariable("PORT", "\${PORT}")
+    }
 }
 
 docker {
     val imageName: String
     val tag = project.version.toString().lowercase()
-    val javaArgs = mutableListOf("-Xms256m", "-Xmx512m")
+    val javaArgs = mutableListOf("-Xms256m", "-Xmx512m", "-DAPP_LOG_LEVEL=debug")
     if (System.getProperty("standalone") == null) {
         imageName = "sszuev/open-tutor"
     } else {

diff --git a/app-ktor/src/main/kotlin/Main.kt → app-ktor/src/main/kotlin/AppMain.kt b/app-ktor/src/main/kotlin/Main.kt → app-ktor/src/main/kotlin/AppMain.kt
@@ -63,7 +63,7 @@ import org.thymeleaf.templateresolver.ClassLoaderTemplateResolver
 import java.nio.charset.StandardCharsets
 import java.util.Base64
 
-private val logger: ExtLogger = logger("com.gitlab.sszuev.flashcards.MainKt")
+private val logger: ExtLogger = logger("com.gitlab.sszuev.flashcards.AppMainKt")
 
 // use to jetty, not netty, due to exception https://youtrack.jetbrains.com/issue/KTOR-4433
 fun main(args: Array<String>) = io.ktor.server.jetty.EngineMain.main(args)
@@ -92,8 +92,8 @@ fun Application.module(
 
     val keycloakProvider = OAuthServerSettings.OAuth2ServerSettings(
         name = "keycloak",
-        authorizeUrl = "${keycloakConfig.address}/auth/realms/${keycloakConfig.realm}/protocol/openid-connect/auth",
-        accessTokenUrl = "${keycloakConfig.address}/auth/realms/${keycloakConfig.realm}/protocol/openid-connect/token",
+        authorizeUrl = "${keycloakConfig.authorizeAddress}/realms/${keycloakConfig.realm}/protocol/openid-connect/auth",
+        accessTokenUrl = "${keycloakConfig.accessTokenAddress}/realms/${keycloakConfig.realm}/protocol/openid-connect/token",
         clientId = keycloakConfig.clientId,
         clientSecret = keycloakConfig.secret,
         accessTokenRequiresBasicAuth = false,
@@ -233,7 +233,7 @@ private fun thymeleafContent(
     } else {
         mapOf(
             "user" to principal.name(),
-            "keycloakAuthURL" to "${keycloakConfig.address}/auth",
+            "keycloakAuthURL" to keycloakConfig.authorizeAddress,
             "keycloakAppRealm" to keycloakConfig.realm,
             "keycloakAppClient" to keycloakConfig.clientId,
         )
@@ -293,7 +293,8 @@ private fun Application.printGeneralSettings(
             |bootstrap-services             = $bootstrapServices
             |run-config-app-mode            = ${runConfig.mode}
             |run-config-debug-auth          = ${runConfig.auth}
-            |keycloak-address               = ${keycloakConfig.address}
+            |keycloak-authorize-address     = ${keycloakConfig.authorizeAddress}
+            |keycloak-access-token-address  = ${keycloakConfig.accessTokenAddress}
             |keycloak-realm                 = ${keycloakConfig.realm}
             |keycloak-client-id             = ${keycloakConfig.clientId}
             |application-port               = $port

diff --git a/app-ktor/src/main/kotlin/config/KeycloakConfig.kt b/app-ktor/src/main/kotlin/config/KeycloakConfig.kt
@@ -1,17 +1,19 @@
 package com.gitlab.sszuev.flashcards.config
 
-import io.ktor.server.config.*
+import io.ktor.server.config.ApplicationConfig
 
 data class KeycloakConfig(
-    val address: String,
+    val authorizeAddress: String,
+    val accessTokenAddress: String,
     val clientId: String,
     val secret: String,
     val issuer: String,
     val audience: String,
     val realm: String,
 ) {
     constructor(config: ApplicationConfig): this(
-        address = config.property("keycloak.address").getString(),
+        authorizeAddress = config.property("keycloak.authorize-address").getString(),
+        accessTokenAddress = config.property("keycloak.access-token-address").getString(),
         clientId = config.property("keycloak.clientId").getString(),
         realm = config.property("keycloak.realm").getString(),
         secret = config.property("keycloak.jwt.secret").getString(),

diff --git a/app-ktor/src/main/kotlin/config/RunConfig.kt b/app-ktor/src/main/kotlin/config/RunConfig.kt
@@ -13,12 +13,6 @@ data class RunConfig(
         mode = AppMode.valueOf(config.property("run-config.mode").getString().uppercase()),
     )
 
-    init {
-        if (mode == AppMode.PROD) {
-            require(auth.isBlank()) { "No auth expected for prod mode." }
-        }
-    }
-
     fun modeString(): String {
         return mode.name.lowercase()
     }

diff --git a/app-ktor/src/main/resources/application.conf b/app-ktor/src/main/resources/application.conf
@@ -7,18 +7,31 @@ ktor {
     }
 
     application {
-        modules = [ com.gitlab.sszuev.flashcards.MainKt.module ]
+        modules = [ com.gitlab.sszuev.flashcards.AppMainKt.module ]
     }
 }
 
+db-pg {
+    url = """jdbc:postgresql://localhost:5432/flashcards"""
+    url = ${?DB_PG_URL}
+    user = "dev"
+    user = ${?DB_PG_USER}
+    pwd = "dev"
+    pwd = ${?DB_PG_PWD}
+    hikari-pool.pool-size = "8"
+    hikari-pool.keep-alive-time-ms = "150000"
+}
+
 db-mem {
     data-directory = """classpath:/data"""
     data-directory = ${?DATA_DIRECTORY}
 }
 
 keycloak {
-    address = "http://localhost:8081"
-    address = ${?KEYCLOAK_ADDRESS}
+    authorize-address = "http://localhost:8081"
+    authorize-address = ${?KEYCLOAK_AUTHORIZE_ADDRESS}
+    access-token-address = "http://localhost:8081"
+    access-token-address = ${?KEYCLOAK_ACCESS_TOKEN_ADDRESS}
     clientId = "flashcards-client"
     realm = "flashcards-realm"
     jwt {

diff --git a/app-ktor/src/test/kotlin/TestUtils.kt b/app-ktor/src/test/kotlin/TestUtils.kt
@@ -22,7 +22,8 @@ import io.ktor.server.testing.testApplication
 import java.util.Base64
 
 val testKeycloakConfig = KeycloakConfig(
-    address = "http://test-keycloak-server.ex",
+    authorizeAddress = "http://test-keycloak-server.ex",
+    accessTokenAddress = "http://test-keycloak-server.ex",
     clientId = "test-client",
     secret = "test-secret",
     issuer = "test-issuer",

diff --git a/db-mem/src/main/kotlin/MemDbCardRepository.kt b/db-mem/src/main/kotlin/MemDbCardRepository.kt
@@ -10,7 +10,7 @@ import com.gitlab.sszuev.flashcards.systemNow
 class MemDbCardRepository(
     dbConfig: MemDbConfig = MemDbConfig(),
 ) : DbCardRepository {
-    private val database = MemDatabase.get(dbConfig.dataLocation)
+    private val database by lazy { MemDatabase.get(dbConfig.dataLocation) }
 
     override fun findCardById(cardId: String): DbCard? =
         database.findCardById(require(cardId.isNotBlank()).run { cardId.toLong() })?.toDbCard()

diff --git a/db-mem/src/main/kotlin/MemDbDictionaryRepository.kt b/db-mem/src/main/kotlin/MemDbDictionaryRepository.kt
@@ -9,7 +9,7 @@ class MemDbDictionaryRepository(
     dbConfig: MemDbConfig = MemDbConfig(),
 ) : DbDictionaryRepository {
 
-    private val database = MemDatabase.get(databaseLocation = dbConfig.dataLocation)
+    private val database by lazy { MemDatabase.get(databaseLocation = dbConfig.dataLocation) }
 
     override fun findDictionaryById(dictionaryId: String): DbDictionary? =
         database.findDictionaryById(dictionaryId.toLong())?.toDbDictionary()

diff --git a/tts-client/src/main/kotlin/TTSClientSettings.kt b/tts-client/src/main/kotlin/TTSClientSettings.kt
@@ -42,16 +42,4 @@ object TTSClientSettings {
             |message-status-header          = $messageStatusHeader
             """.replaceIndentByMargin("\t")
     }
-
-    private fun Config.get(key: String, default: String): String {
-        return if (hasPath(key)) getString(key) else default
-    }
-
-    private fun Config.get(key: String, default: Int): Int {
-        return if (hasPath(key)) getInt(key) else default
-    }
-
-    private fun Config.get(key: String, default: Long): Long {
-        return if (hasPath(key)) getLong(key) else default
-    }
 }
diff --git a/tts-client/src/main/kotlin/rabbitmq/RMQTTSResourceRepository.kt b/tts-client/src/main/kotlin/rabbitmq/RMQTTSResourceRepository.kt
@@ -14,7 +14,15 @@ import com.gitlab.sszuev.flashcards.speaker.TTSClientSettings
 import com.rabbitmq.client.AMQP
 import com.rabbitmq.client.Channel
 import com.rabbitmq.client.Delivery
-import kotlinx.coroutines.*
+import kotlinx.coroutines.CoroutineName
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Deferred
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.asCoroutineDispatcher
+import kotlinx.coroutines.async
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withTimeout
 import org.slf4j.LoggerFactory
 import java.util.concurrent.Executors
 
@@ -58,8 +66,16 @@ class RMQTTSResourceRepository(
     override suspend fun getResource(id: TTSResourceId): TTSResourceEntityResponse {
         val errors: MutableList<AppError> = mutableListOf()
         val entity = try {
-            val res: Deferred<ByteArray> = scope.async(context = Dispatchers.IO) {
-                return@async retrieveData(id)
+            val res: Deferred<Any> = scope.async(context = Dispatchers.IO) {
+                return@async try {
+                    if (logger.isDebugEnabled) {
+                        logger.debug("Get resource id = {}", id)
+                    }
+                    retrieveData(id)
+                } catch (exception: Exception) {
+                    logger.error("Error while getting resource ID={}", id, exception)
+                    exception
+                }
             }
             val data = if (requestTimeoutInMillis < 0) {
                 res.await()
@@ -68,8 +84,22 @@ class RMQTTSResourceRepository(
                     withTimeout(requestTimeoutInMillis) { res.await() }
                 }
             }
-            ResourceEntity(resourceId = id, data = data)
+            when (data) {
+                is Exception -> {
+                    errors.add(data.asError())
+                    ResourceEntity.DUMMY
+                }
+
+                is ByteArray -> {
+                    ResourceEntity(resourceId = id, data = data)
+                }
+
+                else -> {
+                    throw IllegalStateException("Unexpected data")
+                }
+            }
         } catch (ex: Exception) {
+            logger.error("Error while getting resource ID={}", id, ex)
             errors.add(ex.asError())
             ResourceEntity.DUMMY
         }

diff --git a/tts-client/src/main/kotlin/rabbitmq/SimpleRabbitmqConnectionFactory.kt b/tts-client/src/main/kotlin/rabbitmq/SimpleRabbitmqConnectionFactory.kt
@@ -29,7 +29,7 @@ class SimpleRabbitmqConnectionFactory(val config: ConnectionConfig) : RabbitmqCo
                 try {
                     res.close(599, "server shutdown", 2_000)
                 } catch (ignore: AlreadyClosedException) {
-                    logger.debug("Connection $config already closed.")
+                    logger.debug("Connection {} is already closed.", config)
                 }
             }
         })

diff --git a/tts-client/src/test/kotlin/TTSResourceRepositoryITest.kt b/tts-client/src/test/kotlin/TTSResourceRepositoryITest.kt
@@ -5,21 +5,28 @@ import com.gitlab.sszuev.flashcards.model.domain.TTSResourceId
 import com.gitlab.sszuev.flashcards.repositories.TTSResourceRepository
 import com.gitlab.sszuev.flashcards.speaker.ServerResourceException
 import com.gitlab.sszuev.flashcards.speaker.TTSClientConfig
-import com.rabbitmq.client.*
-import kotlinx.coroutines.ExperimentalCoroutinesApi
+import com.rabbitmq.client.AMQP
+import com.rabbitmq.client.CancelCallback
+import com.rabbitmq.client.Channel
+import com.rabbitmq.client.Connection
+import com.rabbitmq.client.ConnectionFactory
+import com.rabbitmq.client.DeliverCallback
 import kotlinx.coroutines.TimeoutCancellationException
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.test.runTest
-import org.junit.jupiter.api.*
+import org.junit.jupiter.api.AfterAll
+import org.junit.jupiter.api.Assertions
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.Timeout
 import org.slf4j.LoggerFactory
 import org.testcontainers.containers.RabbitMQContainer
 import java.util.concurrent.CyclicBarrier
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
 
 @Timeout(value = 420, unit = TimeUnit.SECONDS)
-@OptIn(ExperimentalCoroutinesApi::class)
 internal class TTSResourceRepositoryITest {
 
     companion object {
@@ -28,7 +35,7 @@ internal class TTSResourceRepositoryITest {
         private val container by lazy {
             RabbitMQContainer("rabbitmq:latest").apply {
                 withExposedPorts(5672, 15672)
-                withUser("guest", "guest")
+                //withUser("guest", "guest")
                 start()
             }
         }

diff --git a/tts-lib/README.md b/tts-lib/README.md
@@ -6,10 +6,10 @@ it accepts resource identifier and returns byte array (audio data, wav).
 The resource identifier must be in the format `lang:word`, where `lang` depends on underlying mechanism,
 but usually it is well-known language tag (`en`, `fr`, etc).  
 
-Currently, there are following three implementations of TTS service :
+Currently, there are the following three implementations of TTS service :
 
-- [LocalTextToSpeechService](/src/main/kotlin/impl/LocalTextToSpeechService.kt) - for testing, 
-can work with local tar archives in the shtooka format. See also https://shtooka.net/ - a collection of audio-resources.
+- [LocalTextToSpeechService](/src/main/kotlin/impl/LocalTextToSpeechService.kt) - for testing;
+  can work with local tar archives in the shtooka format. See also https://shtooka.net/ - a collection of audio-resources.
 
 - [VoicerssTextToSpeechService](/src/main/kotlin/impl/VoicerssTextToSpeechService.kt) - a client for http://www.voicerss.org/ service.   
 to make it work please obtain voice-rss-api key and specify it using vm-option `tts.service.voicerss.key`