Skip to content

Commit

Permalink
Merge pull request #27 from crowdproj/learning
Browse files Browse the repository at this point in the history
complete the learning functionality + full lingvo support
  • Loading branch information
sszuev authored Mar 11, 2024
2 parents e0c4b89 + 2e210f5 commit b10e67c
Show file tree
Hide file tree
Showing 23 changed files with 565 additions and 261 deletions.
12 changes: 6 additions & 6 deletions common/src/commonMain/kotlin/repositories/DbCardRepository.kt
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,8 @@ import com.gitlab.sszuev.flashcards.model.common.AppUserId
import com.gitlab.sszuev.flashcards.model.domain.CardEntity
import com.gitlab.sszuev.flashcards.model.domain.CardFilter
import com.gitlab.sszuev.flashcards.model.domain.CardId
import com.gitlab.sszuev.flashcards.model.domain.CardLearn
import com.gitlab.sszuev.flashcards.model.domain.DictionaryEntity
import com.gitlab.sszuev.flashcards.model.domain.DictionaryId
import com.gitlab.sszuev.flashcards.model.domain.LangId

/**
* Database repository to work with cards.
Expand Down Expand Up @@ -40,9 +39,9 @@ interface DbCardRepository {
fun updateCard(userId: AppUserId, cardEntity: CardEntity): CardDbResponse

/**
* Updates cards details.
* Performs bulk update.
*/
fun learnCards(userId: AppUserId, cardLearns: List<CardLearn>): CardsDbResponse
fun updateCards(userId: AppUserId, cardIds: Iterable<CardId>, update: (CardEntity) -> CardEntity): CardsDbResponse

/**
* Resets status.
Expand All @@ -57,11 +56,12 @@ interface DbCardRepository {

data class CardsDbResponse(
val cards: List<CardEntity> = emptyList(),
val sourceLanguageId: LangId = LangId.NONE,
val dictionaries: List<DictionaryEntity> = emptyList(),
val errors: List<AppError> = emptyList(),
) {

companion object {
val EMPTY = CardsDbResponse(cards = emptyList(), errors = emptyList())
val EMPTY = CardsDbResponse(cards = emptyList(), dictionaries = emptyList(), errors = emptyList())
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import com.gitlab.sszuev.flashcards.model.common.AppUserId
import com.gitlab.sszuev.flashcards.model.domain.CardEntity
import com.gitlab.sszuev.flashcards.model.domain.CardFilter
import com.gitlab.sszuev.flashcards.model.domain.CardId
import com.gitlab.sszuev.flashcards.model.domain.CardLearn
import com.gitlab.sszuev.flashcards.model.domain.DictionaryId

object NoOpDbCardRepository : DbCardRepository {
Expand All @@ -28,7 +27,11 @@ object NoOpDbCardRepository : DbCardRepository {
noOp()
}

override fun learnCards(userId: AppUserId, cardLearns: List<CardLearn>): CardsDbResponse {
override fun updateCards(
userId: AppUserId,
cardIds: Iterable<CardId>,
update: (CardEntity) -> CardEntity
): CardsDbResponse {
noOp()
}

Expand Down
33 changes: 15 additions & 18 deletions core/src/main/kotlin/processes/CardProcessWorkers.kt
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import com.gitlab.sszuev.flashcards.corlib.ChainDSL
import com.gitlab.sszuev.flashcards.corlib.worker
import com.gitlab.sszuev.flashcards.model.common.AppStatus
import com.gitlab.sszuev.flashcards.model.domain.CardOperation
import com.gitlab.sszuev.flashcards.model.domain.LangId
import com.gitlab.sszuev.flashcards.model.domain.TTSResourceGet
import com.gitlab.sszuev.flashcards.model.domain.TTSResourceId
import com.gitlab.sszuev.flashcards.repositories.CardDbResponse
Expand Down Expand Up @@ -108,10 +107,7 @@ fun ChainDSL<CardContext>.processLearnCards() = worker {
this.status == AppStatus.RUN
}
process {
val userId = this.contextUserEntity.id
val res =
this.repositories.cardRepository(this.workMode).learnCards(userId, this.normalizedRequestCardLearnList)
this.postProcess(res)
this.postProcess(this.learnCards())
}
onException {
this.handleThrowable(CardOperation.LEARN_CARDS, it)
Expand Down Expand Up @@ -149,21 +145,22 @@ fun ChainDSL<CardContext>.processDeleteCard() = worker {
}

private suspend fun CardContext.postProcess(res: CardsDbResponse) {
if (res.errors.isNotEmpty()) {
this.errors.addAll(res.errors)
}
if (res.sourceLanguageId != LangId.NONE) {
val tts = this.repositories.ttsClientRepository(this.workMode)
this.responseCardEntityList = res.cards.map { card ->
val words = card.words.map { word ->
val audio = tts.findResourceId(TTSResourceGet(word.word, res.sourceLanguageId).normalize())
this.errors.addAll(audio.errors)
if (audio.id != TTSResourceId.NONE) word.copy(sound = audio.id) else word
check(res != CardsDbResponse.EMPTY) { "Null response" }
this.errors.addAll(res.errors)
val sourceLangByDictionary = res.dictionaries.associate { it.dictionaryId to it.sourceLang.langId }
val tts = this.repositories.ttsClientRepository(this.workMode)
this.responseCardEntityList = res.cards.map { card ->
val sourceLang = sourceLangByDictionary[card.dictionaryId] ?: return@map card
val words = card.words.map { word ->
val audio = tts.findResourceId(TTSResourceGet(word.word, sourceLang).normalize())
this.errors.addAll(audio.errors)
if (audio.id != TTSResourceId.NONE) {
word.copy(sound = audio.id)
} else {
word
}
card.copy(words = words)
}
} else {
this.responseCardEntityList = res.cards
card.copy(words = words)
}
this.status = if (this.errors.isNotEmpty()) AppStatus.FAIL else AppStatus.RUN
}
Expand Down
102 changes: 99 additions & 3 deletions core/src/main/kotlin/processes/SearchCardsHelper.kt
Original file line number Diff line number Diff line change
@@ -1,12 +1,108 @@
package com.gitlab.sszuev.flashcards.core.processes

import com.gitlab.sszuev.flashcards.CardContext
import com.gitlab.sszuev.flashcards.model.domain.CardEntity
import com.gitlab.sszuev.flashcards.model.domain.CardWordEntity
import com.gitlab.sszuev.flashcards.repositories.CardsDbResponse

private val comparator: Comparator<CardEntity> = Comparator<CardEntity> { left, right ->
val la = left.answered ?: 0
val ra = right.answered ?: 0
la.compareTo(ra)
}.thenComparing { lc, rc ->
lc.changedAt.compareTo(rc.changedAt)
}

/**
* Prepares a card deck for a tutor-session.
*/
fun CardContext.findCardDeck(): CardsDbResponse {
val userId = this.contextUserEntity.id
return this.repositories.cardRepository(this.workMode).searchCard(userId, this.normalizedRequestCardFilter)
}
return if (this.normalizedRequestCardFilter.random) {
// For random mode, do not use the database support since logic is quite complicated.
// We load everything into memory, since the dictionary can hardly contain more than a thousand words,
// i.e., this is relatively small data.
val filter = this.normalizedRequestCardFilter.copy(random = false, length = -1)
val res = this.repositories.cardRepository(this.workMode).searchCard(this.contextUserEntity.id, filter)
if (res.errors.isNotEmpty()) {
return res
}
var cards = res.cards.shuffled().sortedWith(comparator)
if (this.normalizedRequestCardFilter.length > 0) {
val set = mutableSetOf<CardEntity>()
collectCardDeck(cards, set, this.normalizedRequestCardFilter.length)
cards = set.toList()
cards = cards.shuffled()
}
return res.copy(cards = cards)
} else {
this.repositories.cardRepository(this.workMode)
.searchCard(this.contextUserEntity.id, this.normalizedRequestCardFilter)
}
}

private fun collectCardDeck(all: List<CardEntity>, res: MutableSet<CardEntity>, num: Int) {
if (all.size <= num) {
res.addAll(all)
return
}
val rest = mutableListOf<CardEntity>()
all.forEach { candidate ->
if (isSimilar(candidate, res)) {
rest.add(candidate)
} else {
res.add(candidate)
if (res.size == num) {
return
}
}
}
res.addAll(rest.sortedWith(comparator).take(num - res.size))
}

internal fun isSimilar(candidate: CardEntity, res: Set<CardEntity>): Boolean = res.any { it.isSimilar(candidate) }

internal fun CardEntity.isSimilar(other: CardEntity): Boolean =
if (this == other) true else this.words.isSimilar(other.words)

private fun List<CardWordEntity>.isSimilar(other: List<CardWordEntity>): Boolean {
forEach { left ->
other.forEach { right ->
if (left.isSimilar(right)) {
return true
}
}
}
return false
}

private fun CardWordEntity.isSimilar(other: CardWordEntity): Boolean {
if (this == other) {
return true
}
if (word.isSimilar(other.word)) {
return true
}
val otherTranslations = other.translations.flatten()
translations.flatten().forEach { left ->
otherTranslations.forEach { right ->
if (left.isSimilar(right)) {
return true
}
}
}
return false
}

private fun String.isSimilar(other: String): Boolean {
if (this == other) {
return true
}
return prefix(3) == other.prefix(3)
}

private fun String.prefix(num: Int): String {
if (length <= num) {
return lowercase()
}
return substring(0, num).lowercase()
}
22 changes: 22 additions & 0 deletions core/src/main/kotlin/processes/UpdateCardsHelper.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package com.gitlab.sszuev.flashcards.core.processes

import com.gitlab.sszuev.flashcards.CardContext
import com.gitlab.sszuev.flashcards.repositories.CardsDbResponse

fun CardContext.learnCards(): CardsDbResponse {
val cards = this.normalizedRequestCardLearnList.associateBy { it.cardId }
return this.repositories.cardRepository(this.workMode).updateCards(this.contextUserEntity.id, cards.keys) { card ->
val learn = checkNotNull(cards[card.cardId])
var answered = card.answered?.toLong() ?: 0L
val details = card.stats.toMutableMap()
learn.details.forEach {
answered += it.value.toInt()
require(answered < Int.MAX_VALUE && answered > Int.MIN_VALUE)
if (answered < 0) {
answered = 0
}
details.merge(it.key, it.value) { a, b -> a + b }
}
card.copy(stats = details, answered = answered.toInt())
}
}
10 changes: 6 additions & 4 deletions core/src/main/kotlin/validators/CardValidateWorkers.kt
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ fun ChainDSL<CardContext>.validateCardId(getCardId: (CardContext) -> CardId) = w

fun ChainDSL<CardContext>.validateCardEntityWords(getCard: (CardContext) -> CardEntity) = worker {
this.name = "Test card-word"
test { getCard(this).words.any { !isCorrectWrong(it.word) } }
test {
getCard(this).words.any { !isCorrectWrong(it.word) }
}
process {
fail(validationError(fieldName = "card-word"))
}
Expand All @@ -47,7 +49,7 @@ fun ChainDSL<CardContext>.validateCardFilterLength(getCardFilter: (CardContext)
getCardFilter(this).length <= 0
}
process {
fail(validationError(fieldName = "card-filter-length", description = "must be greater zero"))
fail(validationError(fieldName = "card-filter-length", description = "must be greater than zero"))
}
}

Expand Down Expand Up @@ -89,8 +91,8 @@ fun ChainDSL<CardContext>.validateCardLearnListDetails(getCardLearn: (CardContex
}
}
) { (_, score) ->
// right not just check score is positive and not big
score <= 0 || score > 42
// right not just check score is not big
score < -42 || score > 42
}

private fun ChainDSL<CardContext>.validateIds(
Expand Down
18 changes: 7 additions & 11 deletions core/src/test/kotlin/CardCorProcessorRunCardsTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ import org.junit.jupiter.api.Test
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.EnumSource

@Suppress("OPT_IN_USAGE")
internal class CardCorProcessorRunCardsTest {
companion object {
private val testUser = AppUserEntity(AppUserId("42"), AppAuthId("00000000-0000-0000-0000-000000000000"))
Expand Down Expand Up @@ -225,10 +224,6 @@ internal class CardCorProcessorRunCardsTest {
val context = testContext(CardOperation.CREATE_CARD, repository)
context.requestCardEntity = testRequestEntity

context.errors.forEach { // TODO
println(it)
}

CardCorProcessor().execute(context)

Assertions.assertTrue(wasCalled)
Expand All @@ -241,7 +236,7 @@ context.errors.forEach { // TODO
fun `test search-cards success`() = runTest {
val testFilter = CardFilter(
dictionaryIds = listOf(DictionaryId("21"), DictionaryId("42")),
random = true,
random = false,
withUnknown = true,
length = 42,
)
Expand Down Expand Up @@ -358,10 +353,10 @@ context.errors.forEach { // TODO

var wasCalled = false
val repository = MockDbCardRepository(
invokeLearnCards = { _, it ->
invokeUpdateCards = { _, givenIds, _ ->
wasCalled = true
CardsDbResponse(
cards = if (it == testLearn) testResponseEntities else emptyList(),
cards = if (givenIds == setOf(stubCard.cardId)) testResponseEntities else emptyList(),
)
}
)
Expand All @@ -384,6 +379,7 @@ context.errors.forEach { // TODO
CardLearn(cardId = CardId("1"), details = mapOf(Stage.SELF_TEST to 42)),
CardLearn(cardId = CardId("2"), details = mapOf(Stage.OPTIONS to 2, Stage.MOSAIC to 3))
)
val ids = testLearn.map { it.cardId }.toSet()

val testResponseEntities = stubCards
val testResponseErrors = listOf(
Expand All @@ -392,11 +388,11 @@ context.errors.forEach { // TODO

var wasCalled = false
val repository = MockDbCardRepository(
invokeLearnCards = { _, it ->
invokeUpdateCards = { _, givenIds, _ ->
wasCalled = true
CardsDbResponse(
cards = if (it == testLearn) testResponseEntities else emptyList(),
errors = if (it == testLearn) testResponseErrors else emptyList()
cards = if (givenIds == ids) testResponseEntities else emptyList(),
errors = if (givenIds == ids) testResponseErrors else emptyList()
)
}
)
Expand Down
15 changes: 12 additions & 3 deletions core/src/test/kotlin/CardCorProcessorValidationTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,16 @@ import com.gitlab.sszuev.flashcards.model.common.AppAuthId
import com.gitlab.sszuev.flashcards.model.common.AppError
import com.gitlab.sszuev.flashcards.model.common.AppMode
import com.gitlab.sszuev.flashcards.model.common.AppRequestId
import com.gitlab.sszuev.flashcards.model.domain.*
import com.gitlab.sszuev.flashcards.model.domain.CardEntity
import com.gitlab.sszuev.flashcards.model.domain.CardFilter
import com.gitlab.sszuev.flashcards.model.domain.CardId
import com.gitlab.sszuev.flashcards.model.domain.CardLearn
import com.gitlab.sszuev.flashcards.model.domain.CardOperation
import com.gitlab.sszuev.flashcards.model.domain.CardWordEntity
import com.gitlab.sszuev.flashcards.model.domain.DictionaryId
import com.gitlab.sszuev.flashcards.model.domain.LangId
import com.gitlab.sszuev.flashcards.model.domain.Stage
import com.gitlab.sszuev.flashcards.model.domain.TTSResourceGet
import com.gitlab.sszuev.flashcards.stubs.stubCard
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.test.runTest
Expand All @@ -15,7 +24,7 @@ import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.Arguments
import org.junit.jupiter.params.provider.EnumSource
import org.junit.jupiter.params.provider.MethodSource
import java.util.*
import java.util.UUID

@OptIn(ExperimentalCoroutinesApi::class)
internal class CardCorProcessorValidationTest {
Expand Down Expand Up @@ -314,7 +323,7 @@ internal class CardCorProcessorValidationTest {
val context = testContext(CardOperation.LEARN_CARDS)
context.requestCardLearnList = listOf(
testCardLearn.copy(cardId = CardId("42"), details = mapOf(Stage.OPTIONS to 4200, Stage.WRITING to 42)),
testCardLearn.copy(cardId = CardId("21"), details = mapOf(Stage.MOSAIC to -12, Stage.SELF_TEST to 0)),
testCardLearn.copy(cardId = CardId("21"), details = mapOf(Stage.MOSAIC to -4200, Stage.SELF_TEST to -4200)),
)
processor.execute(context)
Assertions.assertEquals(3, context.errors.size)
Expand Down
Loading

0 comments on commit b10e67c

Please sign in to comment.