Merge remote-tracking branch 'origin/staging' into optimize-periodic-…

…deployment-manger-db-queries
TouK · Dec 19, 2024 · 1c4b171 · 1c4b171
2 parents dbadf80 + 433ec63
commit 1c4b171
Show file tree

Hide file tree

Showing 19 changed files with 270 additions and 58 deletions.
diff --git a/build.sbt b/build.sbt
@@ -1130,7 +1130,7 @@ lazy val mathUtils = (project in utils("math-utils"))
       "org.springframework" % "spring-expression" % springV,
     )
   )
-  .dependsOn(componentsApi, testUtils % Test)
+  .dependsOn(commonUtils, componentsApi, testUtils % Test)
 
 lazy val defaultHelpers = (project in utils("default-helpers"))
   .settings(commonSettings)

diff --git a/designer/client/src/components/graph/node-modal/fragment-input-definition/item/types.ts b/designer/client/src/components/graph/node-modal/fragment-input-definition/item/types.ts
@@ -86,6 +86,7 @@ export function isPermittedTypeVariant(item: FragmentInputParameter): item is Pe
         resolveRefClazzName(item.typ.refClazzName) === "String",
         resolveRefClazzName(item.typ.refClazzName) === "Boolean",
         resolveRefClazzName(item.typ.refClazzName) === "Long",
+        resolveRefClazzName(item.typ.refClazzName) === "Integer",
     ].includes(true);
 }
 

diff --git a/...c/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoaderFactory.scala b/...c/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoaderFactory.scala
@@ -9,7 +9,8 @@ trait ProcessingTypeConfigsLoaderFactory {
 
   def create(
       configLoaderConfig: Config,
+      processingTypeConfigFromDesignerConfig: Config,
       sttpBackend: SttpBackend[IO, Any],
-  )(implicit ec: IORuntime): ProcessingTypeConfigsLoader
+  )(implicit ioRuntime: IORuntime): ProcessingTypeConfigsLoader
 
 }
diff --git a/...odel/src/main/scala/pl/touk/nussknacker/restmodel/validation/PrettyValidationErrors.scala b/...odel/src/main/scala/pl/touk/nussknacker/restmodel/validation/PrettyValidationErrors.scala
@@ -187,19 +187,20 @@ object PrettyValidationErrors {
         )
       case UnsupportedFixedValuesType(paramName, typ, _) =>
         node(
-          message = s"Fixed values list can only be be provided for type String or Boolean, found: $typ",
+          message = s"Fixed values list can only be be provided for type String, Integer, Long or Boolean, found: $typ",
           description = "Please check component definition",
           paramName = Some(qualifiedParamFieldName(paramName = paramName, subFieldName = Some(TypFieldName)))
         )
       case UnsupportedDictParameterEditorType(paramName, typ, _) =>
         node(
-          s"Dictionary parameter editor can only be used for parameters of type String, Long or Boolean, found: $typ",
-          "Please check component definition",
+          message =
+            s"Dictionary parameter editor can only be used for parameters of type String, Integer, Long or Boolean, found: $typ",
+          description = "Please check component definition",
           paramName = Some(qualifiedParamFieldName(paramName = paramName, subFieldName = Some(TypFieldName)))
         )
       case EmptyFixedListForRequiredField(paramName, _) =>
         node(
-          s"Non-empty fixed list of values have to be declared for required parameter",
+          message = s"Non-empty fixed list of values have to be declared for required parameter",
           description = "Please add a value to the list of possible values",
           paramName = Some(qualifiedParamFieldName(paramName = paramName, subFieldName = Some(InputModeFieldName)))
         )

diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfig.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfig.scala
@@ -12,14 +12,14 @@ final case class DesignerConfig private (rawConfig: ConfigWithUnresolvedVersion)
   import net.ceedubs.ficus.Ficus._
 
   def processingTypeConfigs: ProcessingTypeConfigs =
-    ProcessingTypeConfigs(
-      rawConfig
-        .readMap("scenarioTypes")
-        .getOrElse {
-          throw new RuntimeException("No scenario types configuration provided")
-        }
-        .mapValuesNow(ProcessingTypeConfig.read)
-    )
+    ProcessingTypeConfigs(processingTypeConfigsRaw.asMap.mapValuesNow(ProcessingTypeConfig.read))
+
+  def processingTypeConfigsRaw: ConfigWithUnresolvedVersion =
+    rawConfig
+      .getConfigOpt("scenarioTypes")
+      .getOrElse {
+        throw new RuntimeException("No scenario types configuration provided")
+      }
 
   def configLoaderConfig: Config = rawConfig.resolved.getAs[Config]("configLoader").getOrElse(ConfigFactory.empty())
 

diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala
@@ -78,7 +78,7 @@ class NussknackerAppFactory(
         logger.debug(
           s"Found custom ${classOf[ProcessingTypeConfigsLoaderFactory].getSimpleName}: ${factory.getClass.getName}. Using it for configuration loading"
         )
-        factory.create(designerConfig.configLoaderConfig, sttpBackend)
+        factory.create(designerConfig.configLoaderConfig, designerConfig.processingTypeConfigsRaw.resolved, sttpBackend)
       }
       .getOrElse {
         logger.debug(

diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DictApiHttpServiceSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DictApiHttpServiceSpec.scala
@@ -31,8 +31,14 @@ class DictApiHttpServiceSpec
         .post(s"$nuDesignerHttpAddress/api/processDefinitionData/${Streaming.stringify}/dicts")
         .Then()
         .statusCode(200)
-        .equalsJsonBody("[]")
-
+        .equalsJsonBody(
+          s"""[
+             |  {
+             |    "id" : "integer_dict",
+             |    "label" : "integer_dict"
+             |  }
+             |]""".stripMargin
+        )
     }
 
     "return proper list for expected type String" in {
@@ -83,6 +89,10 @@ class DictApiHttpServiceSpec
         .statusCode(200)
         .equalsJsonBody(
           s"""[
+             |  {
+             |    "id": "integer_dict",
+             |    "label": "integer_dict"
+             |  },
              |  {
              |    "id" : "long_dict",
              |    "label" : "long_dict"
@@ -91,7 +101,7 @@ class DictApiHttpServiceSpec
         )
     }
 
-    "return proper list for expected type BigDecimal" in {
+    "return proper list for expected type BigInteger" in {
       given()
         .when()
         .basicAuthAllPermUser()
@@ -107,6 +117,11 @@ class DictApiHttpServiceSpec
         .statusCode(200)
         .equalsJsonBody(
           s"""[
+             |
+             |  {
+             |    "id": "integer_dict",
+             |    "label": "integer_dict"
+             |  },
              |  {
              |    "id" : "long_dict",
              |    "label" : "long_dict"

diff --git a/docs/Changelog.md b/docs/Changelog.md
@@ -29,6 +29,9 @@
 * [#7332](https://github.com/TouK/nussknacker/pull/7332) Handle scenario names with spaces when performing migration tests, they were ignored
 * [#7346](https://github.com/TouK/nussknacker/pull/7346) OpenAPI enricher: ability to configure common secret for any security scheme
 * [#7307](https://github.com/TouK/nussknacker/pull/7307) Added StddevPop, StddevSamp, VarPop and VarSamp aggregators
+* [#7341](https://github.com/TouK/nussknacker/pull/7341) Added possibility to choose presets and define lists for Integer typed parameter inputs in fragments.
+* [#7356](https://github.com/TouK/nussknacker/pull/7356) Integers converted to BigDecimals have scale 18,
+  this fixes issue with unexpected low scale when performing division on BigDecimals which were created in such conversion.
 
 ## 1.18
 

diff --git a/.../executor/src/test/scala/pl/touk/nussknacker/engine/process/functional/FragmentSpec.scala b/.../executor/src/test/scala/pl/touk/nussknacker/engine/process/functional/FragmentSpec.scala
@@ -10,18 +10,42 @@ import pl.touk.nussknacker.engine.canonicalgraph.{CanonicalProcess, canonicalnod
 import pl.touk.nussknacker.engine.compile.FragmentResolver
 import pl.touk.nussknacker.engine.graph.evaluatedparam.BranchParameters
 import pl.touk.nussknacker.engine.graph.evaluatedparam.{Parameter => NodeParameter}
+import pl.touk.nussknacker.engine.graph.expression.Expression
 import pl.touk.nussknacker.engine.graph.node.FragmentInputDefinition.{FragmentClazzRef, FragmentParameter}
 import pl.touk.nussknacker.engine.graph.node._
 import pl.touk.nussknacker.engine.graph.sink.SinkRef
+import pl.touk.nussknacker.engine.graph.variable.Field
 import pl.touk.nussknacker.engine.process.helpers.ProcessTestHelpers
 import pl.touk.nussknacker.engine.process.helpers.SampleNodes._
+import pl.touk.nussknacker.springframework.util.BigDecimalScaleEnsurer
 
 import java.util.Date
 
 class FragmentSpec extends AnyFunSuite with Matchers with ProcessTestHelpers {
 
   import pl.touk.nussknacker.engine.spel.SpelExtension._
 
+  test("should properly convert fragment input to BigDecimal") {
+
+    val process = resolve(
+      ScenarioBuilder
+        .streaming("proc1")
+        .source("id", "input")
+        .fragmentOneOut("sub", "fragmentWithBigDecimalInput", "output", "fragmentResult", "param" -> "1".spel)
+        .processorEnd("end1", "logService", "all" -> "#fragmentResult.a".spel)
+    )
+
+    val data = List(
+      SimpleRecord("1", 12, "a", new Date(0))
+    )
+
+    processInvoker.invokeWithSampleData(process, data)
+    val results = ProcessTestHelpers.logServiceResultsHolder.results
+
+    results.size shouldBe 1
+    results(0).asInstanceOf[java.math.BigDecimal].scale() shouldBe BigDecimalScaleEnsurer.DEFAULT_BIG_DECIMAL_SCALE
+  }
+
   test("should accept same id in fragment and main process ") {
 
     val process = resolve(
@@ -101,6 +125,20 @@ class FragmentSpec extends AnyFunSuite with Matchers with ProcessTestHelpers {
   }
 
   private def resolve(scenario: CanonicalProcess) = {
+    val fragmentWithBigDecimalInput = CanonicalProcess(
+      MetaData("fragmentWithBigDecimalInput", FragmentSpecificData()),
+      List(
+        canonicalnode.FlatNode(
+          FragmentInputDefinition(
+            "start",
+            List(FragmentParameter(ParameterName("param"), FragmentClazzRef[java.math.BigDecimal]))
+          )
+        ),
+        canonicalnode.FlatNode(FragmentOutputDefinition("outB1", "output", List(Field("a", Expression.spel("#param")))))
+      ),
+      List.empty
+    )
+
     val fragment = CanonicalProcess(
       MetaData("fragment1", FragmentSpecificData()),
       List(
@@ -182,7 +220,9 @@ class FragmentSpec extends AnyFunSuite with Matchers with ProcessTestHelpers {
     )
 
     val resolved =
-      FragmentResolver(List(fragmentWithSplit, fragment, fragmentWithGlobalVar, diamondFragment)).resolve(scenario)
+      FragmentResolver(
+        List(fragmentWithBigDecimalInput, fragmentWithSplit, fragment, fragmentWithGlobalVar, diamondFragment)
+      ).resolve(scenario)
 
     resolved shouldBe Symbol("valid")
     resolved.toOption.get

diff --git a/...src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala b/...src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala
@@ -242,6 +242,7 @@ class DevProcessConfigCreator extends ProcessConfigCreator {
         BusinessConfigDictionary.id -> BusinessConfigDictionary.definition,
         BooleanDictionary.id -> BooleanDictionary.definition, // not available through global variables, but still available through DictParameterEditor
         LongDictionary.id -> LongDictionary.definition, // not available through global variables, but still available through DictParameterEditor
+        IntegerDictionary.id -> IntegerDictionary.definition // not available through global variables, but still available through DictParameterEditor
       )
     )
   }

diff --git a/.../src/main/scala/pl/touk/nussknacker/engine/management/sample/dict/IntegerDictionary.scala b/.../src/main/scala/pl/touk/nussknacker/engine/management/sample/dict/IntegerDictionary.scala
@@ -0,0 +1,23 @@
+package pl.touk.nussknacker.engine.management.sample.dict
+
+import pl.touk.nussknacker.engine.api.dict.embedded.EmbeddedDictDefinition
+import pl.touk.nussknacker.engine.api.dict.{DictDefinition, DictInstance, ReturningKeyWithoutTransformation}
+import pl.touk.nussknacker.engine.api.typed.typing
+import pl.touk.nussknacker.engine.api.typed.typing.Typed
+
+object IntegerDictionary {
+  val id: String = "integer_dict"
+
+  val definition: DictDefinition = new EmbeddedDictDefinition with ReturningKeyWithoutTransformation {
+    override def valueType(dictId: String): typing.SingleTypingResult = Typed.typedClass[java.lang.Integer]
+
+    override def labelByKey: Map[String, String] = Map(
+      "-2147483648" -> "large (negative) number",
+      "42"          -> "small number",
+      "2147483647"  -> "big number"
+    )
+
+  }
+
+  val instance: DictInstance = DictInstance(id, definition)
+}
diff --git a/...a/pl/touk/nussknacker/engine/lite/components/requestresponse/CollectTransformerTest.scala b/...a/pl/touk/nussknacker/engine/lite/components/requestresponse/CollectTransformerTest.scala
@@ -55,7 +55,7 @@ class CollectTransformerTest
   test("should collect elements after for-each") {
     val scenario = scenarioBuilderWithSchemas
       .customNode("for-each", "outForEach", "for-each", "Elements" -> "#input".spel)
-      .buildSimpleVariable("someVar", "ourVar", "'x = ' + (#outForEach * 2)".spel)
+      .buildSimpleVariable("someVar", "ourVar", "'x = ' + (#outForEach.intValue() * 2)".spel)
       .customNode("collect", "outCollector", "collect", "Input expression" -> "#ourVar".spel)
       .emptySink("response", "response", SinkRawEditorParamName.value -> "true".spel, "Value" -> "#outCollector".spel)
     val requestElements = (0 to 3).toList

diff --git a/extensions-api/src/main/scala/pl/touk/nussknacker/engine/ConfigWithUnresolvedVersion.scala b/extensions-api/src/main/scala/pl/touk/nussknacker/engine/ConfigWithUnresolvedVersion.scala
@@ -10,21 +10,21 @@ case class ConfigWithUnresolvedVersion private (withUnresolvedEnvVariables: Conf
     ConfigWithUnresolvedVersion(withUnresolvedEnvVariables.getConfig(path), resolved.getConfig(path))
   }
 
-  def readMap(path: String): Option[Map[String, ConfigWithUnresolvedVersion]] = {
-    if (resolved.hasPath(path)) {
-      val nestedConfig = getConfig(path)
-      Some(
-        nestedConfig.resolved
-          .root()
-          .entrySet()
-          .asScala
-          .map(_.getKey)
-          .map { key => key -> nestedConfig.getConfig(key) }
-          .toMap
-      )
-    } else {
+  def getConfigOpt(path: String): Option[ConfigWithUnresolvedVersion] = {
+    if (resolved.hasPath(path))
+      Some(ConfigWithUnresolvedVersion(withUnresolvedEnvVariables.getConfig(path), resolved.getConfig(path)))
+    else
       None
-    }
+  }
+
+  def asMap: Map[String, ConfigWithUnresolvedVersion] = {
+    resolved
+      .root()
+      .entrySet()
+      .asScala
+      .map(_.getKey)
+      .map { key => key -> getConfig(key) }
+      .toMap
   }
 
 }

diff --git a/...scala/pl/touk/nussknacker/engine/compile/nodecompilation/FragmentParameterValidator.scala b/...scala/pl/touk/nussknacker/engine/compile/nodecompilation/FragmentParameterValidator.scala
@@ -38,12 +38,14 @@ import scala.util.Try
 object FragmentParameterValidator {
 
   val permittedTypesForEditors: List[FragmentClazzRef] = List(
-    FragmentClazzRef[java.lang.Boolean],
     FragmentClazzRef[String],
+    FragmentClazzRef[java.lang.Boolean],
+    FragmentClazzRef[java.lang.Integer],
     FragmentClazzRef[java.lang.Long],
     FragmentClazzRef("String"),
     FragmentClazzRef("Boolean"),
-    FragmentClazzRef("Long")
+    FragmentClazzRef("Integer"),
+    FragmentClazzRef("Long"),
   )
 
 }

diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/extension/Conversion.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/extension/Conversion.scala
@@ -6,6 +6,7 @@ import org.springframework.util.NumberUtils
 import pl.touk.nussknacker.engine.api.generics.GenericFunctionTypingError
 import pl.touk.nussknacker.engine.api.typed.typing.{Typed, TypingResult}
 import pl.touk.nussknacker.engine.util.classes.Extensions.ClassExtensions
+import pl.touk.nussknacker.springframework.util.BigDecimalScaleEnsurer
 
 import java.lang.{
   Boolean => JBoolean,
@@ -68,11 +69,14 @@ object ToBigDecimalConversion extends ToNumericConversion[JBigDecimal] {
 
   override def convertEither(value: Any): Either[Throwable, JBigDecimal] =
     value match {
-      case v: JBigDecimal => Right(v)
-      case v: JBigInteger => Right(new JBigDecimal(v))
+      case v: JBigDecimal => Right(BigDecimalScaleEnsurer.ensureBigDecimalScale(v))
+      case v: JBigInteger => Right(BigDecimalScaleEnsurer.ensureBigDecimalScale(new JBigDecimal(v)))
       case v: Number      => Try(NumberUtils.convertNumberToTargetClass(v, resultTypeClass)).toEither
-      case v: String      => Try(new JBigDecimal(v)).toEither
-      case _              => Left(new IllegalArgumentException(s"Cannot convert: $value to BigDecimal"))
+      case v: String =>
+        Try({
+          BigDecimalScaleEnsurer.ensureBigDecimalScale(new JBigDecimal(v))
+        }).toEither
+      case _ => Left(new IllegalArgumentException(s"Cannot convert: $value to BigDecimal"))
     }
 
 }