diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/SimpleObjectEncoder.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/SimpleObjectEncoder.scala deleted file mode 100644 index 69442a1e64d..00000000000 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/SimpleObjectEncoder.scala +++ /dev/null @@ -1,47 +0,0 @@ -package pl.touk.nussknacker.engine.api.typed - -import cats.data.{Validated, ValidatedNel} -import cats.implicits.catsSyntaxValidatedId -import io.circe.{ACursor, Decoder, DecodingFailure, Json} -import io.circe.Json.{fromBoolean, fromDouble, fromFloat, fromInt, fromLong, fromString} -import pl.touk.nussknacker.engine.api.typed.typing.{Typed, TypedClass} - -// TODO: Add support for more types. -object SimpleObjectEncoder { - private val intClass = Typed.typedClass[Int] - private val longClass = Typed.typedClass[Long] - private val floatClass = Typed.typedClass[Float] - private val doubleClass = Typed.typedClass[Double] - private val booleanClass = Typed.typedClass[Boolean] - private val stringClass = Typed.typedClass[String] - - def encode(typ: TypedClass, data: Any): ValidatedNel[String, Json] = (typ, data) match { - case (`intClass`, intValue: Int) => - fromInt(intValue).validNel - case (`longClass`, longValue: Long) => - fromLong(longValue).validNel - case (`floatClass`, floatValue: Float) => - fromFloat(floatValue).map(_.validNel).getOrElse(s"Could not encode $floatValue as json.".invalidNel) - case (`doubleClass`, doubleValue: Double) => - fromDouble(doubleValue).map(_.validNel).getOrElse(s"Could not encode $doubleValue as json.".invalidNel) - case (`booleanClass`, booleanValue: Boolean) => - fromBoolean(booleanValue).validNel - case (`stringClass`, stringValue: String) => - fromString(stringValue).validNel - case (klass, value) if value.getClass == klass.klass => - s"No encoding logic for $typ.".invalidNel - case (klass, value) => - s"Mismatched class and value: $klass and $value".invalidNel - } - - def decode(typ: TypedClass, obj: ACursor): Decoder.Result[Any] = typ match { - case `intClass` => obj.as[Int] - case `longClass` => obj.as[Long] - case `floatClass` => obj.as[Float] - case `doubleClass` => obj.as[Double] - case `booleanClass` => obj.as[Boolean] - case `stringClass` => obj.as[String] - case typ => Left(DecodingFailure(s"No decoding logic for $typ.", List())) - } - -} diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/TypeEncoders.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/TypeEncoders.scala index d70db1eb7c1..0904d015195 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/TypeEncoders.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/TypeEncoders.scala @@ -68,9 +68,10 @@ object TypeEncoders { objTypeEncoded.+:(tagEncoded) case TypedObjectWithValue(underlying, value) => val objTypeEncoded = encodeTypingResult(underlying) - val dataEncoded: (String, Json) = "value" -> SimpleObjectEncoder - .encode(underlying, value) + val dataEncoded: (String, Json) = "value" -> ValueEncoder + .encodeValue(value) .getOrElse(throw new IllegalStateException(s"Not supported data value: $value")) + objTypeEncoded.+:(dataEncoded) case cl: TypedClass => encodeTypedClass(cl) } @@ -129,7 +130,7 @@ class TypingResultDecoder(loadClass: String => Class[_]) { private def typedObjectWithValue(obj: HCursor): Decoder.Result[TypingResult] = for { valueClass <- typedClass(obj) - value <- SimpleObjectEncoder.decode(valueClass, obj.downField("value")) + value <- ValueDecoder.decodeValue(valueClass, obj.downField("value")) } yield TypedObjectWithValue(valueClass, value) private def typedObjectTypingResult(obj: HCursor): Decoder.Result[TypingResult] = for { diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/ValueDecoder.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/ValueDecoder.scala new file mode 100644 index 00000000000..0d508eaa878 --- /dev/null +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/ValueDecoder.scala @@ -0,0 +1,63 @@ +package pl.touk.nussknacker.engine.api.typed + +import cats.implicits.toTraverseOps +import io.circe.{ACursor, Decoder, DecodingFailure, Json} +import pl.touk.nussknacker.engine.api.typed.typing._ + +import java.math.BigInteger +import scala.jdk.CollectionConverters._ + +object ValueDecoder { + private val intClass = Typed.typedClass[Int] + private val shortClass = Typed.typedClass[Short] + private val longClass = Typed.typedClass[Long] + private val floatClass = Typed.typedClass[Float] + private val doubleClass = Typed.typedClass[Double] + private val booleanClass = Typed.typedClass[Boolean] + private val stringClass = Typed.typedClass[String] + private val byteClass = Typed.typedClass[Byte] + private val bigIntegerClass = Typed.typedClass[BigInteger] + private val bigDecimalClass = Typed.typedClass[java.math.BigDecimal] + + def decodeValue(typ: TypingResult, obj: ACursor): Decoder.Result[Any] = typ match { + case TypedObjectWithValue(_, value) => Right(value) + case TypedNull => Right(null) + case `intClass` => obj.as[Int] + case `shortClass` => obj.as[Short] + case `longClass` => obj.as[Long] + case `floatClass` => obj.as[Float] + case `doubleClass` => obj.as[Double] + case `booleanClass` => obj.as[Boolean] + case `stringClass` => obj.as[String] + case `byteClass` => obj.as[Byte] + case `bigIntegerClass` => obj.as[BigInteger] + case `bigDecimalClass` => obj.as[java.math.BigDecimal] + case TypedClass(klass, List(elementType: TypingResult)) if klass == classOf[java.util.List[_]] => + obj.values match { + case Some(values) => + values.toList + .traverse(v => decodeValue(elementType, v.hcursor)) + .map(_.asJava) + case None => + Left(DecodingFailure(s"Expected encoded List to be a Json array", List())) + } + case record: TypedObjectTypingResult => + for { + fieldsJson <- obj.as[Map[String, Json]] + decodedFields <- record.fields.toList.traverse { case (fieldName, fieldType) => + fieldsJson.get(fieldName) match { + case Some(fieldJson) => decodeValue(fieldType, fieldJson.hcursor).map(fieldName -> _) + case None => + Left( + DecodingFailure( + s"Record field '$fieldName' isn't present in encoded Record fields: $fieldsJson", + List() + ) + ) + } + } + } yield decodedFields.toMap.asJava + case typ => Left(DecodingFailure(s"Decoding of type [$typ] is not supported.", List())) + } + +} diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/ValueEncoder.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/ValueEncoder.scala new file mode 100644 index 00000000000..51cb8c50230 --- /dev/null +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/ValueEncoder.scala @@ -0,0 +1,48 @@ +package pl.touk.nussknacker.engine.api.typed + +import cats.data.ValidatedNel +import cats.implicits.{catsSyntaxValidatedId, toTraverseOps} +import io.circe.Json +import io.circe.Json.{fromBigDecimal, fromBigInt, fromBoolean, fromDouble, fromFloat, fromInt, fromLong, fromString} + +import java.math.BigInteger +import scala.jdk.CollectionConverters._ + +object ValueEncoder { + + def encodeValue(value: Any): ValidatedNel[String, Json] = value match { + case null => Json.Null.validNel + case value: Int => fromInt(value).validNel + case value: Short => fromInt(value).validNel + case value: Long => fromLong(value).validNel + case value: Boolean => fromBoolean(value).validNel + case value: String => fromString(value).validNel + case value: Byte => fromInt(value).validNel + case value: BigInteger => fromBigInt(value).validNel + case value: java.math.BigDecimal => fromBigDecimal(value).validNel + case value: Float => + fromFloat(value).map(_.validNel).getOrElse(s"Could not encode $value as json.".invalidNel) + case value: Double => + fromDouble(value).map(_.validNel).getOrElse(s"Could not encode $value as json.".invalidNel) + + case vals: java.util.Collection[_] => + val encodedValues = vals.asScala.map(elem => encodeValue(elem)).toList.sequence + encodedValues.map(values => Json.fromValues(values)) + case vals: java.util.Map[_, _] => + val encodedFields = vals.asScala.toList.map { case (key, value) => + encodeValue(key).andThen(encodedKey => + encodedKey.asString match { + case Some(encodedKeyString) => + encodeValue(value).map(encodedValue => encodedKeyString -> encodedValue) + case None => + s"Failed to encode Record key '$encodedKey' as String".invalidNel + } + ) + } + + encodedFields.sequence.map(values => Json.fromFields(values)) + case value => + s"Encoding of value [$value] of class [${value.getClass.getName}] is not supported".invalidNel + } + +} diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/typing.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/typing.scala index 08917cb0332..0665a294d0b 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/typing.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/typed/typing.scala @@ -71,8 +71,8 @@ object typing { objType: TypedClass, additionalInfo: Map[String, AdditionalDataValue] = Map.empty ) extends SingleTypingResult { - override def valueOpt: Option[Map[String, Any]] = - fields.map { case (k, v) => v.valueOpt.map((k, _)) }.toList.sequence.map(Map(_: _*)) + override def valueOpt: Option[java.util.Map[String, Any]] = + fields.map { case (k, v) => v.valueOpt.map((k, _)) }.toList.sequence.map(Map(_: _*).asJava) override def withoutValue: TypedObjectTypingResult = TypedObjectTypingResult(fields.mapValuesNow(_.withoutValue), objType, additionalInfo) @@ -116,12 +116,16 @@ object typing { override def withoutValue: SingleTypingResult = underlying.withoutValue - override def display: String = { - val dataString = value.toString - val shortenedDataString = - if (dataString.length <= maxDataDisplaySize) dataString - else dataString.take(maxDataDisplaySizeWithDots) ++ "..." - s"${underlying.display}($shortenedDataString)" + override def display: String = s"${underlying.display}($shortenedDataString)" + + private def shortenedDataString = { + val dataString = value match { + case l: java.util.List[_] => l.asScala.mkString("{", ", ", "}") + case _ => value.toString + } + + if (dataString.length <= maxDataDisplaySize) dataString + else dataString.take(maxDataDisplaySizeWithDots) ++ "..." } } @@ -257,6 +261,15 @@ object typing { cl } + def typedListWithElementValues[T]( + elementType: TypingResult, + elementValues: java.util.List[T] + ): TypedObjectWithValue = + TypedObjectWithValue( + Typed.genericTypeClass(classOf[java.util.List[_]], List(elementType)), + elementValues + ) + private def toRuntime[T: ClassTag]: Class[_] = implicitly[ClassTag[T]].runtimeClass // parameters - None if you are not in generic aware context, Some - otherwise @@ -328,12 +341,16 @@ object typing { case list: List[_] => genericTypeClass(classOf[List[_]], List(supertypeOfElementTypes(list))) case javaList: java.util.List[_] => - genericTypeClass(classOf[java.util.List[_]], List(supertypeOfElementTypes(javaList.asScala.toList))) + typedListWithElementValues( + supertypeOfElementTypes(javaList.asScala.toList).withoutValue, + javaList + ) case typeFromInstance: TypedFromInstance => typeFromInstance.typingResult + // TODO: handle more types, for example Set case other => Typed(other.getClass) match { case typedClass: TypedClass => - SimpleObjectEncoder.encode(typedClass, other) match { + ValueEncoder.encodeValue(other) match { case Valid(_) => TypedObjectWithValue(typedClass, other) case Invalid(_) => typedClass } diff --git a/components-api/src/test/scala/pl/touk/nussknacker/engine/api/typed/TypedFromInstanceTest.scala b/components-api/src/test/scala/pl/touk/nussknacker/engine/api/typed/TypedFromInstanceTest.scala index 783dbe2b161..6e8e4ce6708 100644 --- a/components-api/src/test/scala/pl/touk/nussknacker/engine/api/typed/TypedFromInstanceTest.scala +++ b/components-api/src/test/scala/pl/touk/nussknacker/engine/api/typed/TypedFromInstanceTest.scala @@ -4,7 +4,8 @@ import org.scalatest.LoneElement import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks -import pl.touk.nussknacker.engine.api.typed.typing._ +import pl.touk.nussknacker.engine.api.typed.typing.Typed.typedListWithElementValues +import pl.touk.nussknacker.engine.api.typed.typing.{Unknown, _} class TypedFromInstanceTest extends AnyFunSuite with Matchers with LoneElement with TableDrivenPropertyChecks { @@ -68,12 +69,20 @@ class TypedFromInstanceTest extends AnyFunSuite with Matchers with LoneElement w val typingResult = Typed.fromInstance(obj) typingResult.canBeSubclassOf(Typed(klass)) shouldBe true - typingResult.asInstanceOf[TypedClass].params.loneElement.canBeSubclassOf(paramTypingResult) shouldBe true + typingResult.withoutValue + .asInstanceOf[TypedClass] + .params + .loneElement + .canBeSubclassOf(paramTypingResult) shouldBe true } def checkNotASubclassOfOtherParamTypingResult(obj: Any, otherParamTypingResult: TypingResult): Unit = { val typingResult = Typed.fromInstance(obj) - typingResult.asInstanceOf[TypedClass].params.loneElement.canBeSubclassOf(otherParamTypingResult) shouldBe false + typingResult.withoutValue + .asInstanceOf[TypedClass] + .params + .loneElement + .canBeSubclassOf(otherParamTypingResult) shouldBe false } val listOfSimpleObjects = List[Any](1.1, 2) @@ -98,7 +107,7 @@ class TypedFromInstanceTest extends AnyFunSuite with Matchers with LoneElement w ) } - test("should find element type for lists of different elements") { + test("should find element type for scala lists of different elements") { Typed.fromInstance(List[Any](4L, 6.35, 8.47)) shouldBe Typed.genericTypeClass( classOf[List[_]], List(Typed.typedClass[Number]) @@ -106,6 +115,20 @@ class TypedFromInstanceTest extends AnyFunSuite with Matchers with LoneElement w Typed.fromInstance(List(3, "t")) shouldBe Typed.genericTypeClass(classOf[List[_]], List(Unknown)) } + test("should find element type and keep values for java lists of different elements") { + val numberList = List(4L, 6.35, 8.47).asJava + Typed.fromInstance(numberList) shouldBe typedListWithElementValues( + Typed.typedClass[Double], + numberList + ) + + val anyList = List(3, "t").asJava + Typed.fromInstance(anyList) shouldBe typedListWithElementValues( + Unknown, + anyList + ) + } + test("should fallback to object's class") { Typed.fromInstance("abc") shouldBe TypedObjectWithValue(Typed.typedClass[String], "abc") } diff --git a/components-api/src/test/scala/pl/touk/nussknacker/engine/api/typed/TypingResultDecoderSpec.scala b/components-api/src/test/scala/pl/touk/nussknacker/engine/api/typed/TypingResultDecoderSpec.scala index 1f91d41f11a..f783a652868 100644 --- a/components-api/src/test/scala/pl/touk/nussknacker/engine/api/typed/TypingResultDecoderSpec.scala +++ b/components-api/src/test/scala/pl/touk/nussknacker/engine/api/typed/TypingResultDecoderSpec.scala @@ -4,9 +4,12 @@ import com.typesafe.scalalogging.LazyLogging import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import pl.touk.nussknacker.engine.api.typed.typing.Typed.typedListWithElementValues import pl.touk.nussknacker.engine.api.typed.typing._ import pl.touk.nussknacker.test.EitherValuesDetailedMessage +import scala.jdk.CollectionConverters._ + class TypingResultDecoderSpec extends AnyFunSuite with ScalaCheckDrivenPropertyChecks @@ -42,9 +45,20 @@ class TypingResultDecoderSpec Map("field1" -> Typed[String]), Typed.typedClass[Map[String, Any]], Map[String, AdditionalDataValue]("ad1" -> "aaa", "ad2" -> 22L, "ad3" -> true) + ), + typedListWithElementValues(Typed[Int], List(1, 2, 3).asJava), + typedListWithElementValues(Typed[String], List("foo", "bar").asJava), + typedListWithElementValues(Typed.record(Map.empty), List(Map.empty.asJava).asJava), + typedListWithElementValues( + Typed.record( + Map("a" -> TypedObjectWithValue(Typed.typedClass[Int], 1)) + ), + List(Map("a" -> 1).asJava).asJava ) ).foreach { typing => - decoder.decodeTypingResults.decodeJson(TypeEncoders.typingResultEncoder(typing)).rightValue shouldBe typing + val encoded = TypeEncoders.typingResultEncoder(typing) + + decoder.decodeTypingResults.decodeJson(encoded).rightValue shouldBe typing } } diff --git a/components-api/src/test/scala/pl/touk/nussknacker/engine/api/typed/ValueDecoderSpec.scala b/components-api/src/test/scala/pl/touk/nussknacker/engine/api/typed/ValueDecoderSpec.scala new file mode 100644 index 00000000000..5c7a4205012 --- /dev/null +++ b/components-api/src/test/scala/pl/touk/nussknacker/engine/api/typed/ValueDecoderSpec.scala @@ -0,0 +1,107 @@ +package pl.touk.nussknacker.engine.api.typed + +import com.typesafe.scalalogging.LazyLogging +import io.circe.Json +import io.circe.syntax.EncoderOps +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import pl.touk.nussknacker.engine.api.typed.typing._ +import pl.touk.nussknacker.test.EitherValuesDetailedMessage + +import scala.jdk.CollectionConverters._ + +class ValueDecoderSpec extends AnyFunSuite with EitherValuesDetailedMessage with Matchers with LazyLogging { + + test("decodeValue should decode Record fields correctly when all fields are present") { + val typedRecord = Typed.record( + Map( + "name" -> Typed.fromInstance("Alice"), + "age" -> Typed.fromInstance(30) + ) + ) + + val json = Json.obj( + "name" -> "Alice".asJson, + "age" -> 30.asJson + ) + + ValueDecoder.decodeValue(typedRecord, json.hcursor) shouldEqual Right( + Map( + "name" -> "Alice", + "age" -> 30 + ).asJava + ) + } + + test("decodeValue should fail when a required Record field is missing") { + val typedRecord = Typed.record( + Map( + "name" -> Typed.fromInstance("Alice"), + "age" -> Typed.fromInstance(30) + ) + ) + + val json = Json.obj( + "name" -> "Alice".asJson + ) + + ValueDecoder.decodeValue(typedRecord, json.hcursor).leftValue.message should include( + "Record field 'age' isn't present in encoded Record fields" + ) + } + + test("decodeValue should not include extra fields that aren't typed") { + val typedRecord = Typed.record( + Map( + "name" -> Typed.fromInstance("Alice"), + "age" -> Typed.fromInstance(30) + ) + ) + + val json = Json.obj( + "name" -> "Alice".asJson, + "age" -> 30.asJson, + "occupation" -> "nurse".asJson, + ) + + ValueDecoder.decodeValue(typedRecord, json.hcursor) shouldEqual Right( + Map( + "name" -> "Alice", + "age" -> 30 + ).asJava + ) + } + + test("decodeValue should handle nested records correctly") { + val json = Json.obj( + "name" -> "Alice".asJson, + "address" -> Json.obj( + "street" -> "Main St".asJson, + "houseNumber" -> "123".asJson + ) + ) + + val typedRecord = Typed.record( + Map( + "name" -> Typed.fromInstance("Alice"), + "address" -> Typed.record( + Map( + "street" -> Typed.fromInstance("Main St"), + "houseNumber" -> Typed.fromInstance("123") + ) + ) + ) + ) + + ValueDecoder.decodeValue(typedRecord, json.hcursor) shouldEqual Right( + Map( + "name" -> "Alice", + "address" -> Map( + "street" -> "Main St", + "houseNumber" -> "123" + ).asJava + ).asJava + ) + } + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/NodesApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/NodesApiEndpoints.scala index b2048b9e36a..1f720ad987b 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/NodesApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/NodesApiEndpoints.scala @@ -1492,7 +1492,7 @@ object NodesApiEndpoints { } -object TypingDtoSchemas { +object TypingDtoSchemas { // todo import pl.touk.nussknacker.engine.api.typed.typing._ import sttp.tapir.Schema.SName diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/validation/UIProcessValidatorSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/validation/UIProcessValidatorSpec.scala index 017552bbe32..6f4f81f6297 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/validation/UIProcessValidatorSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/validation/UIProcessValidatorSpec.scala @@ -85,7 +85,15 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP import UIProcessValidatorSpec._ import pl.touk.nussknacker.engine.spel.SpelExtension._ - private val validationExpression = s"#${ValidationExpressionParameterValidator.variableName}.length() < 7" + private val validationExpression = + Expression.spel(s"#${ValidationExpressionParameterValidator.variableName}.length() < 7") + + private val validationExpressionForRecord = + Expression.spel( + s"{'valid','otherValid'}.contains(#${ValidationExpressionParameterValidator.variableName}.get('field'))" + ) + + private val validationExpressionForList = Expression.spel(s"#value.size() == 2 && #value[0] == 'foo'") test("check for not unique edge types") { val process = createGraph( @@ -1078,26 +1086,15 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP ) { val process = processWithEagerServiceWithDynamicComponent("") - val validator = new UIProcessValidator( - processingType = "Streaming", - validator = ProcessValidator.default( - LocalModelData( - ConfigWithScalaVersion.StreamingProcessTypeConfig.resolved.getConfig("modelConfig"), - List(ComponentDefinition("eagerServiceWithDynamicComponent", EagerServiceWithDynamicComponent)), - additionalConfigsFromProvider = Map( - DesignerWideComponentId("streaming-service-eagerServiceWithDynamicComponent") -> ComponentAdditionalConfig( - parameterConfigs = Map( - ParameterName("param") -> ParameterAdditionalUIConfig(required = true, None, None, None, None) - ) - ) + val validator = validatorWithComponentsAndConfig( + List(ComponentDefinition("eagerServiceWithDynamicComponent", EagerServiceWithDynamicComponent)), + Map( + DesignerWideComponentId("streaming-service-eagerServiceWithDynamicComponent") -> ComponentAdditionalConfig( + parameterConfigs = Map( + ParameterName("param") -> ParameterAdditionalUIConfig(required = true, None, None, None, None) ) ) - ), - scenarioProperties = Map.empty, - scenarioPropertiesConfigFinalizer = - new ScenarioPropertiesConfigFinalizer(TestAdditionalUIConfigProvider, Streaming.stringify), - additionalValidators = List.empty, - fragmentResolver = new FragmentResolver(new StubFragmentRepository(Map.empty)) + ) ) val result = validator.validate(process, ProcessTestData.sampleProcessName, isFragment = false) @@ -1125,26 +1122,15 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP ) { val process = processWithOptionalParameterService("") - val validator = new UIProcessValidator( - processingType = "Streaming", - validator = ProcessValidator.default( - LocalModelData( - ConfigWithScalaVersion.StreamingProcessTypeConfig.resolved.getConfig("modelConfig"), - List(ComponentDefinition("optionalParameterService", OptionalParameterService)), - additionalConfigsFromProvider = Map( - DesignerWideComponentId("streaming-service-optionalParameterService") -> ComponentAdditionalConfig( - parameterConfigs = Map( - ParameterName("optionalParam") -> ParameterAdditionalUIConfig(required = true, None, None, None, None) - ) - ) + val validator = validatorWithComponentsAndConfig( + List(ComponentDefinition("optionalParameterService", OptionalParameterService)), + Map( + DesignerWideComponentId("streaming-service-optionalParameterService") -> ComponentAdditionalConfig( + parameterConfigs = Map( + ParameterName("optionalParam") -> ParameterAdditionalUIConfig(required = true, None, None, None, None) ) ) - ), - scenarioProperties = Map.empty, - scenarioPropertiesConfigFinalizer = - new ScenarioPropertiesConfigFinalizer(TestAdditionalUIConfigProvider, Streaming.stringify), - additionalValidators = List.empty, - fragmentResolver = new FragmentResolver(new StubFragmentRepository(Map.empty)) + ) ) val result = validator.validate(process, ProcessTestData.sampleProcessName, isFragment = false) @@ -1172,34 +1158,15 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP ) { val process = processWithOptionalParameterService("'Barabasz'") - val validator = new UIProcessValidator( - processingType = "Streaming", - validator = ProcessValidator.default( - LocalModelData( - ConfigWithScalaVersion.StreamingProcessTypeConfig.resolved.getConfig("modelConfig"), - List(ComponentDefinition("optionalParameterService", OptionalParameterService)), - additionalConfigsFromProvider = Map( - DesignerWideComponentId("streaming-service-optionalParameterService") -> ComponentAdditionalConfig( - parameterConfigs = Map( - ParameterName("optionalParam") -> ParameterAdditionalUIConfig( - required = false, - initialValue = None, - hintText = None, - valueEditor = None, - valueCompileTimeValidation = Some( - ParameterValueCompileTimeValidation(validationExpression.spel, Some("some custom failure message")) - ) - ) - ) - ) + val validator = validatorWithComponentsAndConfig( + List(ComponentDefinition("optionalParameterService", OptionalParameterService)), + Map( + DesignerWideComponentId("streaming-service-optionalParameterService") -> ComponentAdditionalConfig( + parameterConfigs = Map( + ParameterName("optionalParam") -> paramConfigWithValidationExpression(validationExpression) ) ) - ), - scenarioProperties = Map.empty, - scenarioPropertiesConfigFinalizer = - new ScenarioPropertiesConfigFinalizer(TestAdditionalUIConfigProvider, Streaming.stringify), - additionalValidators = List.empty, - fragmentResolver = new FragmentResolver(new StubFragmentRepository(Map.empty)) + ) ) val result = validator.validate(process, ProcessTestData.sampleProcessName, isFragment = false) @@ -1227,34 +1194,15 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP ) { val process = processWithEagerServiceWithDynamicComponent("'Barabasz'") - val validator = new UIProcessValidator( - processingType = "Streaming", - validator = ProcessValidator.default( - LocalModelData( - ConfigWithScalaVersion.StreamingProcessTypeConfig.resolved.getConfig("modelConfig"), - List(ComponentDefinition("eagerServiceWithDynamicComponent", EagerServiceWithDynamicComponent)), - additionalConfigsFromProvider = Map( - DesignerWideComponentId("streaming-service-eagerServiceWithDynamicComponent") -> ComponentAdditionalConfig( - parameterConfigs = Map( - ParameterName("param") -> ParameterAdditionalUIConfig( - required = false, - initialValue = None, - hintText = None, - valueEditor = None, - valueCompileTimeValidation = Some( - ParameterValueCompileTimeValidation(validationExpression.spel, Some("some custom failure message")) - ) - ) - ) - ) + val validator = validatorWithComponentsAndConfig( + List(ComponentDefinition("eagerServiceWithDynamicComponent", EagerServiceWithDynamicComponent)), + Map( + DesignerWideComponentId("streaming-service-eagerServiceWithDynamicComponent") -> ComponentAdditionalConfig( + parameterConfigs = Map( + ParameterName("param") -> paramConfigWithValidationExpression(validationExpression) ) ) - ), - scenarioProperties = Map.empty, - scenarioPropertiesConfigFinalizer = - new ScenarioPropertiesConfigFinalizer(TestAdditionalUIConfigProvider, Streaming.stringify), - additionalValidators = List.empty, - fragmentResolver = new FragmentResolver(new StubFragmentRepository(Map.empty)) + ) ) val result = validator.validate(process, ProcessTestData.sampleProcessName, isFragment = false) @@ -1277,6 +1225,104 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP result.warnings shouldBe ValidationWarnings.success } + test( + "validate Map service parameter based on additional config from provider - ValidationExpressionParameterValidator" + ) { + val process = processWithService( + MapParameterService.serviceId, + List( + NodeParameter( + ParameterName("mapParam1"), + "{'field': 'valid'}".spel + ), + NodeParameter( + ParameterName("mapParam2"), + "{'field': 'invalid'}".spel + ), + ) + ) + + val validator = validatorWithComponentsAndConfig( + List(ComponentDefinition(MapParameterService.serviceId, MapParameterService)), + Map( + DesignerWideComponentId(s"streaming-service-${MapParameterService.serviceId}") -> ComponentAdditionalConfig( + parameterConfigs = Map( + ParameterName("mapParam1") -> paramConfigWithValidationExpression(validationExpressionForRecord), + ParameterName("mapParam2") -> paramConfigWithValidationExpression(validationExpressionForRecord) + ) + ) + ) + ) + + val result = validator.validate(process, ProcessTestData.sampleProcessName, isFragment = false) + + result.errors.globalErrors shouldBe empty + result.errors.invalidNodes.get("custom") should matchPattern { + case Some( + List( + NodeValidationError( + "CustomParameterValidationError", + "some custom failure message", + "Please provide value that satisfies the validation expression '{'valid','otherValid'}.contains(#value.get('field'))'", + Some("mapParam2"), + NodeValidationErrorType.SaveAllowed, + None + ) + ) + ) => + } + result.warnings shouldBe ValidationWarnings.success + } + + test( + "validate List service parameter based on additional config from provider - ValidationExpressionParameterValidator" + ) { + val process = processWithService( + ListParameterService.serviceId, + List( + NodeParameter( + ParameterName("listParam1"), + "{'foo', 'bar'}".spel + ), + NodeParameter( + ParameterName("listParam2"), + "{'bar'}".spel + ), + ) + ) + + val validator = validatorWithComponentsAndConfig( + List(ComponentDefinition(ListParameterService.serviceId, ListParameterService)), + Map( + DesignerWideComponentId(s"streaming-service-${ListParameterService.serviceId}") -> ComponentAdditionalConfig( + parameterConfigs = Map( + ParameterName("listParam1") -> paramConfigWithValidationExpression(validationExpressionForList), + ParameterName("listParam2") -> paramConfigWithValidationExpression(validationExpressionForList) + ) + ) + ) + ) + + val result = validator.validate(process, ProcessTestData.sampleProcessName, isFragment = false) + + result.errors.globalErrors shouldBe empty + result.errors.invalidNodes.get("custom") should matchPattern { + case Some( + List( + NodeValidationError( + "CustomParameterValidationError", + "some custom failure message", + "Please provide value that satisfies the validation expression '#value.size() == 2 && #value[0] == 'foo''", + Some("listParam2"), + NodeValidationErrorType.SaveAllowed, + None + ) + ) + ) => + } + result.warnings shouldBe ValidationWarnings.success + } + test("validate service parameter based on input config - MandatoryParameterValidator") { val validator = new UIProcessValidator( processingType = "Streaming", @@ -1337,7 +1383,7 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP "validationExpression" -> fromMap( Map( "language" -> "spel", - "expression" -> validationExpression + "expression" -> validationExpression.expression ).asJava ), "validationFailedMessage" -> "some custom failure message", @@ -1378,24 +1424,8 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP result.warnings shouldBe ValidationWarnings.success } - private def procesWithDictParameterEditorService(expression: Expression) = createGraph( - List( - Source("inID", SourceRef(existingSourceFactory, List())), - Enricher( - "custom", - ServiceRef( - dictParameterEditorServiceId, - List(NodeParameter(ParameterName("expression"), expression)) - ), - "out" - ), - Sink("out", SinkRef(existingSinkFactory, List())) - ), - List(Edge("inID", "custom", None), Edge("custom", "out", None)) - ) - test("reports expression parsing error in DictParameterEditor") { - val process = procesWithDictParameterEditorService( + val process = processWithDictParameterEditorService( Expression( Language.DictKeyWithLabel, "not parsable key with label expression" @@ -1423,7 +1453,7 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP } test("checks for unknown dictId in DictParameterEditor") { - val process = procesWithDictParameterEditorService(Expression.dictKeyWithLabel("someKey", Some("someLabel"))) + val process = processWithDictParameterEditorService(Expression.dictKeyWithLabel("someKey", Some("someLabel"))) val result = processValidatorWithDicts(Map.empty).validate(process, sampleProcessName, isFragment = false) @@ -1447,7 +1477,7 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP test("checks for unknown key in DictParameterEditor") { val process = - procesWithDictParameterEditorService(Expression.dictKeyWithLabel("thisKeyDoesntExist", Some("someLabel"))) + processWithDictParameterEditorService(Expression.dictKeyWithLabel("thisKeyDoesntExist", Some("someLabel"))) val result = processValidatorWithDicts( Map("someDictId" -> EmbeddedDictDefinition(Map.empty)) @@ -1472,7 +1502,7 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP } test("validate DictParameterEditor happy path") { - val process = procesWithDictParameterEditorService(Expression.dictKeyWithLabel("someKey", Some("someLabel"))) + val process = processWithDictParameterEditorService(Expression.dictKeyWithLabel("someKey", Some("someLabel"))) val result = processValidatorWithDicts( Map("someDictId" -> EmbeddedDictDefinition(Map("someKey" -> "someLabel"))) @@ -1702,12 +1732,7 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP initialValue = None, hintText = None, valueEditor = None, - valueCompileTimeValidation = Some( - ParameterValueCompileTimeValidation( - validationExpression.spel, - None - ) - ) + valueCompileTimeValidation = Some(ParameterValueCompileTimeValidation(validationExpression, None)) ) ) ) @@ -1743,10 +1768,7 @@ class UIProcessValidatorSpec extends AnyFunSuite with Matchers with TableDrivenP hintText = None, valueEditor = None, valueCompileTimeValidation = Some( - ParameterValueCompileTimeValidation( - validationExpression.spel, - Some("some failed message") - ) + ParameterValueCompileTimeValidation(validationExpression, Some("some failed message")) ) ) ) @@ -2133,6 +2155,34 @@ private object UIProcessValidatorSpec { } + object MapParameterService extends Service { + + val serviceId = "mapParameterService" + + @MethodToInvoke + def method( + @ParamName("mapParam1") + mapParam1: Option[java.util.Map[String, String]], + @ParamName("mapParam2") + mapParam2: Option[java.util.Map[String, String]] + ): Future[String] = ??? + + } + + object ListParameterService extends Service { + + val serviceId = "listParameterService" + + @MethodToInvoke + def method( + @ParamName("listParam1") + listParam1: Option[java.util.List[String]], + @ParamName("listParam2") + listParam2: Option[java.util.List[String]] + ): Future[String] = ??? + + } + object EagerServiceWithDynamicComponent extends EagerServiceWithStaticParametersAndReturnType { override def parameters: List[Parameter] = List( @@ -2218,6 +2268,48 @@ private object UIProcessValidatorSpec { ) } + private def processWithDictParameterEditorService(expression: Expression) = + processWithService(dictParameterEditorServiceId, List(NodeParameter(ParameterName("expression"), expression))) + + private def processWithService(serviceId: String, params: List[NodeParameter]) = createGraph( + List( + Source("inID", SourceRef(existingSourceFactory, List())), + Enricher("custom", ServiceRef(serviceId, params), "out"), + Sink("out", SinkRef(existingSinkFactory, List())) + ), + List(Edge("inID", "custom", None), Edge("custom", "out", None)) + ) + + private def paramConfigWithValidationExpression(validationExpression: Expression) = + ParameterAdditionalUIConfig( + required = false, + initialValue = None, + hintText = None, + valueEditor = None, + valueCompileTimeValidation = Some( + ParameterValueCompileTimeValidation(validationExpression, Some("some custom failure message")) + ) + ) + + private def validatorWithComponentsAndConfig( + components: List[ComponentDefinition], + additionalConfigsFromProvider: Map[DesignerWideComponentId, ComponentAdditionalConfig] + ) = new UIProcessValidator( + processingType = "Streaming", + validator = ProcessValidator.default( + LocalModelData( + ConfigWithScalaVersion.StreamingProcessTypeConfig.resolved.getConfig("modelConfig"), + components, + additionalConfigsFromProvider = additionalConfigsFromProvider + ) + ), + scenarioProperties = Map.empty, + scenarioPropertiesConfigFinalizer = + new ScenarioPropertiesConfigFinalizer(TestAdditionalUIConfigProvider, Streaming.stringify), + additionalValidators = List.empty, + fragmentResolver = new FragmentResolver(new StubFragmentRepository(Map.empty)) + ) + def mockedProcessValidator( fragmentInDefaultProcessingType: Option[CanonicalProcess], execConfig: Config = ConfigFactory.empty() diff --git a/docs/Changelog.md b/docs/Changelog.md index b6adfdb500c..2e3e735b737 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -31,6 +31,8 @@ * [#6195](https://github.com/TouK/nussknacker/pull/6195) Added randomized test data generation for Table Source scenarios * [#6340](https://github.com/TouK/nussknacker/pull/6340) Added running tests on generated data for Table Source scenarios * [#6445](https://github.com/TouK/nussknacker/pull/6445) [#6499](https://github.com/TouK/nussknacker/pull/6499) Add support to seconds in a duration editor +* [#6436](https://github.com/TouK/nussknacker/pull/6436) Typed SpEL list expressions will now infer their compile-time known values, instead of only the supertype of its elements. These values can be used in custom components or validators. + * NOTE: selection (`.?`), projection (`.!`) or operations from the `#COLLECTIONS` helper cause the typed list to lose its elements' values 1.16.2 (18 July 2024) ------------------------- diff --git a/docs/MigrationGuide.md b/docs/MigrationGuide.md index 7a1778f3b25..23db9c141b2 100644 --- a/docs/MigrationGuide.md +++ b/docs/MigrationGuide.md @@ -29,6 +29,11 @@ To see the biggest differences please consult the [changelog](Changelog.md). * `EmitWatermarkAfterEachElementCollectionSource.create` takes `ClassTag` implicit parameter instead of `TypeInformation` * `CollectionSource`'s `TypeInformation` implicit parameter was removed * `EmptySource`'s `TypeInformation` implicit parameter was removed +* [#6436](https://github.com/TouK/nussknacker/pull/6436) Changes to `TypingResult` of SpEL expressions that are maps or lists: + * `TypedObjectTypingResult.valueOpt` now returns a `java.util.Map` instead of `scala.collection.immutable.Map` + * NOTE: selection (`.?`) or operations from the `#COLLECTIONS` helper cause the map to lose track of its keys/values, reverting its `fields` to an empty Map + * SpEL list expression are now typed as `TypedObjectWithValue`, with the `underlying` `TypedClass` equal to the `TypedClass` before this change, and with `value` equal to a `java.util.List` of the elements' values. + * NOTE: selection (`.?`), projection (`.!`) or operations from the `#COLLECTIONS` helper cause the list to lose track of its values, reverting it to a value-less `TypedClass` like before the change ### REST API changes diff --git a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/typeinformation/TypingResultAwareTypeInformationDetection.scala b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/typeinformation/TypingResultAwareTypeInformationDetection.scala index 0bc1d27ff92..a7ae3843deb 100644 --- a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/typeinformation/TypingResultAwareTypeInformationDetection.scala +++ b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/typeinformation/TypingResultAwareTypeInformationDetection.scala @@ -122,6 +122,8 @@ class TypingResultAwareTypeInformationDetection(customisation: TypingResultAware case a: SingleTypingResult if a.objType.params.isEmpty => TypeInformation.of(a.objType.klass) // TODO: how can we handle union - at least of some types? + case TypedObjectWithValue(tc: TypedClass, _) => + forType(tc) case _ => TypeInformation.of(classOf[Any]) }).asInstanceOf[TypeInformation[T]] diff --git a/engine/lite/components/kafka-tests/src/test/scala/pl/touk/nussknacker/engine/lite/components/LiteKafkaUniversalAvroSchemaFunctionalTest.scala b/engine/lite/components/kafka-tests/src/test/scala/pl/touk/nussknacker/engine/lite/components/LiteKafkaUniversalAvroSchemaFunctionalTest.scala index e16a9ec5773..98b5c67a4ad 100644 --- a/engine/lite/components/kafka-tests/src/test/scala/pl/touk/nussknacker/engine/lite/components/LiteKafkaUniversalAvroSchemaFunctionalTest.scala +++ b/engine/lite/components/kafka-tests/src/test/scala/pl/touk/nussknacker/engine/lite/components/LiteKafkaUniversalAvroSchemaFunctionalTest.scala @@ -453,7 +453,7 @@ class LiteKafkaUniversalAvroSchemaFunctionalTest ), ( rConfig(sampleInteger, recordIntegerSchema, recordArrayOfNumbersSchema, List(sampleString)), - invalidTypes(s"path 'field[]' actual: '${typedStr.display}' expected: 'Integer | Double'") + invalidTypes(s"path 'field[]' actual: '${typedStr.withoutValue.display}' expected: 'Integer | Double'") ), // FIXME: List[Unknown] (rConfig(sampleInteger, recordIntegerSchema, recordWithArrayOfNumbers, s"""{$sampleBoolean, "$sampleString"}"""), invalidTypes(s"path 'field[]' actual: '${typeBool.display} | ${typeStr.display}' expected: 'Integer | Double'")), ( @@ -474,7 +474,7 @@ class LiteKafkaUniversalAvroSchemaFunctionalTest ), ( rConfig(sampleInteger, recordIntegerSchema, recordMaybeArrayOfNumbersSchema, List(sampleString)), - invalidTypes(s"path 'field[]' actual: '${typedStr.display}' expected: 'Integer | Double'") + invalidTypes(s"path 'field[]' actual: '${typedStr.withoutValue.display}' expected: 'Integer | Double'") ), // FIXME: List[Unknown] (rConfig(sampleInteger, recordIntegerSchema, recordWithMaybeArrayOfNumbers, s"""{$sampleBoolean, "$sampleString"}"""), invalidTypes("path 'field[]' actual: '${typeBool.display} | ${typeStr.display}' expected: 'Integer | Double'")), ( @@ -496,7 +496,7 @@ class LiteKafkaUniversalAvroSchemaFunctionalTest // FIXME: List[Unknown] (rConfig(sampleInteger, recordIntegerSchema, recordWithMaybeArrayOfNumbers, s"""{$sampleBoolean, "$sampleString"}"""), invalidTypes("path 'field[]' actual: '${typeBool.display} | ${typeStr.display}' expected: 'Integer | Double'")), ( rConfig(sampleInteger, recordIntegerSchema, recordMaybeArrayOfNumbersSchema, List(sampleString)), - invalidTypes(s"path 'field[]' actual: '${typedStr.display}' expected: 'Integer | Double'") + invalidTypes(s"path 'field[]' actual: '${typedStr.withoutValue.display}' expected: 'Integer | Double'") ), ( rConfig(sampleInteger, recordIntegerSchema, recordOptionalArrayOfNumbersSchema, sampleInteger), @@ -535,11 +535,13 @@ class LiteKafkaUniversalAvroSchemaFunctionalTest recordOptionalArrayOfArraysNumbersSchema, List(List(sampleString)) ), - invalidTypes(s"path 'field[][]' actual: '${typedStr.display}' expected: 'Integer | Double'") + invalidTypes(s"path 'field[][]' actual: '${typedStr.withoutValue.display}' expected: 'Integer | Double'") ), ( rConfig(sampleInteger, recordIntegerSchema, recordOptionalArrayOfArraysNumbersSchema, List(sampleInteger)), - invalidTypes(s"path 'field[]' actual: '${typedInt.display}' expected: 'Null | List[Integer | Double]'") + invalidTypes( + s"path 'field[]' actual: '${typedInt.withoutValue.display}' expected: 'Null | List[Integer | Double]'" + ) ), ( rConfig(sampleInteger, recordIntegerSchema, recordOptionalArrayOfArraysNumbersSchema, sampleInteger), @@ -573,7 +575,7 @@ class LiteKafkaUniversalAvroSchemaFunctionalTest recordOptionalArrayOfRecordsSchema, List(Map("price" -> sampleString)) ), - invalidTypes(s"path 'field[].price' actual: '${typedStr.display}' expected: 'Null | Double'") + invalidTypes(s"path 'field[].price' actual: '${typedStr.withoutValue.display}' expected: 'Null | Double'") ), ( rConfig(sampleInteger, recordIntegerSchema, recordOptionalArrayOfRecordsSchema, sampleInteger), @@ -584,7 +586,7 @@ class LiteKafkaUniversalAvroSchemaFunctionalTest ( rConfig(sampleInteger, recordIntegerSchema, recordOptionalArrayOfRecordsSchema, List(sampleInteger)), invalidTypes( - s"""path 'field[]' actual: '${typedInt.display}' expected: 'Null | Record{price: Null | Double}'""" + s"""path 'field[]' actual: '${typedInt.withoutValue.display}' expected: 'Null | Record{price: Null | Double}'""" ) ), ) @@ -613,7 +615,7 @@ class LiteKafkaUniversalAvroSchemaFunctionalTest ), ( rConfig(sampleInteger, recordIntegerSchema, recordMapOfIntsSchema, Nil), - invalidTypes("path 'field' actual: 'List[Unknown]' expected: 'Map[String,Null | Integer]'") + invalidTypes("path 'field' actual: 'List[Unknown]({})' expected: 'Map[String,Null | Integer]'") ), ( rConfig(sampleInteger, recordIntegerSchema, recordMapOfIntsSchema, null), @@ -633,7 +635,7 @@ class LiteKafkaUniversalAvroSchemaFunctionalTest ), ( rConfig(sampleInteger, recordIntegerSchema, recordMaybeMapOfIntsSchema, Nil), - invalidTypes("path 'field' actual: 'List[Unknown]' expected: 'Null | Map[String,Null | Integer]'") + invalidTypes("path 'field' actual: 'List[Unknown]({})' expected: 'Null | Map[String,Null | Integer]'") ), ( rConfig( @@ -668,7 +670,7 @@ class LiteKafkaUniversalAvroSchemaFunctionalTest ( rConfig(sampleInteger, recordIntegerSchema, recordOptionalMapOfMapsIntsSchema, Nil), invalidTypes( - "path 'field' actual: 'List[Unknown]' expected: 'Null | Map[String,Null | Map[String,Null | Integer]]'" + "path 'field' actual: 'List[Unknown]({})' expected: 'Null | Map[String,Null | Map[String,Null | Integer]]'" ) ), ( @@ -716,7 +718,7 @@ class LiteKafkaUniversalAvroSchemaFunctionalTest ( rConfig(sampleInteger, recordIntegerSchema, recordOptionalMapOfRecordsSchema, Nil), invalidTypes( - "path 'field' actual: 'List[Unknown]' expected: 'Null | Map[String,Null | Record{price: Null | Double}]'" + "path 'field' actual: 'List[Unknown]({})' expected: 'Null | Map[String,Null | Record{price: Null | Double}]'" ) ), ( @@ -782,7 +784,7 @@ class LiteKafkaUniversalAvroSchemaFunctionalTest ( rConfig(sampleInteger, recordIntegerSchema, nestedRecordSchema, Nil), invalidTypes( - "path 'field' actual: 'List[Unknown]' expected: 'Null | Record{sub: Null | Record{price: Null | Double}}'" + "path 'field' actual: 'List[Unknown]({})' expected: 'Null | Record{sub: Null | Record{price: Null | Double}}'" ) ), ( diff --git a/engine/lite/components/kafka-tests/src/test/scala/pl/touk/nussknacker/engine/lite/components/LiteKafkaUniversalJsonFunctionalTest.scala b/engine/lite/components/kafka-tests/src/test/scala/pl/touk/nussknacker/engine/lite/components/LiteKafkaUniversalJsonFunctionalTest.scala index 7f105d4fb13..cbb495da6d7 100644 --- a/engine/lite/components/kafka-tests/src/test/scala/pl/touk/nussknacker/engine/lite/components/LiteKafkaUniversalJsonFunctionalTest.scala +++ b/engine/lite/components/kafka-tests/src/test/scala/pl/touk/nussknacker/engine/lite/components/LiteKafkaUniversalJsonFunctionalTest.scala @@ -35,7 +35,7 @@ import pl.touk.nussknacker.engine.util.test.TestScenarioRunner.RunnerListResult import pl.touk.nussknacker.test.{SpecialSpELElement, ValidatedValuesDetailedMessage} class LiteKafkaUniversalJsonFunctionalTest - extends AnyFunSuite + extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyChecks with Inside @@ -50,8 +50,8 @@ class LiteKafkaUniversalJsonFunctionalTest import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap import pl.touk.nussknacker.test.LiteralSpELImplicits._ - private val lax = List(ValidationMode.lax) - private val strict = List(ValidationMode.strict) + private val lax = List(ValidationMode.lax) + private val strict = List(ValidationMode.strict) private val strictAndLax = ValidationMode.values test("should test end to end kafka json data at sink and source / handling nulls and empty json") { @@ -198,14 +198,14 @@ class LiteKafkaUniversalJsonFunctionalTest forAll(testData) { ( - input: Json, - sourceSchema: EveritSchema, - sinkSchema: EveritSchema, - validationModes: List[ValidationMode], - expected: Validated[_, RunResult] + input: Json, + sourceSchema: EveritSchema, + sinkSchema: EveritSchema, + validationModes: List[ValidationMode], + expected: Validated[_, RunResult] ) => validationModes.foreach { mode => - val cfg = config(input, sourceSchema, sinkSchema, output = Input, Some(mode)) + val cfg = config(input, sourceSchema, sinkSchema, output = Input, Some(mode)) val results = runWithValueResults(cfg) results shouldBe expected } @@ -225,14 +225,14 @@ class LiteKafkaUniversalJsonFunctionalTest forAll(testData) { ( - input: Json, - sourceSchema: EveritSchema, - sinkSchema: EveritSchema, - validationModes: List[ValidationMode], - expected: Validated[_, RunResult] + input: Json, + sourceSchema: EveritSchema, + sinkSchema: EveritSchema, + validationModes: List[ValidationMode], + expected: Validated[_, RunResult] ) => validationModes.foreach { mode => - val cfg = config(input, sourceSchema, sinkSchema, output = input, Some(mode)) + val cfg = config(input, sourceSchema, sinkSchema, output = input, Some(mode)) val results = runWithValueResults(cfg) results shouldBe expected } @@ -240,7 +240,7 @@ class LiteKafkaUniversalJsonFunctionalTest } test("sink with enum schema") { - val A = Json.fromString("A") + val A = Json.fromString("A") val one = Json.fromInt(1) val two = Json.fromInt(2) val obj = Json.obj(("x", A), ("y", Json.arr(one, two))) @@ -267,14 +267,14 @@ class LiteKafkaUniversalJsonFunctionalTest forAll(testData) { ( - input: Json, - sourceSchema: EveritSchema, - sinkSchema: EveritSchema, - validationModes: List[ValidationMode], - expected: Validated[_, RunResult] + input: Json, + sourceSchema: EveritSchema, + sinkSchema: EveritSchema, + validationModes: List[ValidationMode], + expected: Validated[_, RunResult] ) => validationModes.foreach { mode => - val cfg = config(input, sourceSchema, sinkSchema, output = Input, Some(mode)) + val cfg = config(input, sourceSchema, sinkSchema, output = Input, Some(mode)) val results = runWithValueResults(cfg) results shouldBe expected } @@ -286,10 +286,12 @@ class LiteKafkaUniversalJsonFunctionalTest config(Json.obj(), schemaObjStr, schemaEnumStrOrList, output = SpecialSpELElement("{1,2}"), lax.headOption) val results = runWithValueResults(cfg) - results.isValid shouldBe true // it should be invalid, but it's so edge case that we decided to live with it - val runtimeError = results.validValue.errors.head - runtimeError.nodeId shouldBe Some("my-sink") - runtimeError.throwable.asInstanceOf[RuntimeException].getMessage shouldBe "#: [1,2] is not a valid enum value" + results.isValid shouldBe false + val runtimeError = results.invalidValue.head + runtimeError.nodeIds shouldBe Set("my-sink") + runtimeError.asInstanceOf[CustomNodeError].message shouldBe + """Provided value does not match scenario output - errors: + |Incorrect type: actual: 'List[Integer]({1, 2})' expected: 'List[Integer]({1, 2, 3}) | String(A)'.""".stripMargin } test("patternProperties handling") { @@ -303,10 +305,10 @@ class LiteKafkaUniversalJsonFunctionalTest Map("definedProp" -> schemaString) ) - val inputObjectIntPropValue = fromInt(1) + val inputObjectIntPropValue = fromInt(1) val inputObjectDefinedPropValue = fromString("someString") - val inputObject = obj("foo_int" -> inputObjectIntPropValue) - val inputObjectWithDefinedProp = obj("definedProp" -> inputObjectDefinedPropValue) + val inputObject = obj("foo_int" -> inputObjectIntPropValue) + val inputObjectWithDefinedProp = obj("definedProp" -> inputObjectDefinedPropValue) //@formatter:off val testData = Table( @@ -329,15 +331,15 @@ class LiteKafkaUniversalJsonFunctionalTest forAll(testData) { ( - input: Json, - sourceSchema: EveritSchema, - sinkSchema: EveritSchema, - sinkExpression: SpecialSpELElement, - validationModes: List[ValidationMode], - expected: Validated[_, RunResult] + input: Json, + sourceSchema: EveritSchema, + sinkSchema: EveritSchema, + sinkExpression: SpecialSpELElement, + validationModes: List[ValidationMode], + expected: Validated[_, RunResult] ) => validationModes.foreach { mode => - val cfg = config(input, sourceSchema, sinkSchema, output = sinkExpression, Some(mode)) + val cfg = config(input, sourceSchema, sinkSchema, output = sinkExpression, Some(mode)) val results = runWithValueResults(cfg) results shouldBe expected } @@ -352,7 +354,8 @@ class LiteKafkaUniversalJsonFunctionalTest val objWithNestedPatternPropertiesMapSchema = createObjSchema(true, false, createObjectSchemaWithPatternProperties(Map("_int$" -> schemaLong))) - val objectWithNettedPatternPropertiesMapAsRefSchema = JsonSchemaBuilder.parseSchema("""{ + val objectWithNettedPatternPropertiesMapAsRefSchema = JsonSchemaBuilder.parseSchema( + """{ | "type": "object", | "properties": { | "field": { @@ -395,8 +398,8 @@ class LiteKafkaUniversalJsonFunctionalTest forAll(testData) { (sinkSchema: EveritSchema, sinkFields: Map[String, String], expected: Validated[_, RunResult]) => val dummyInputObject = obj() - val cfg = config(dummyInputObject, schemaMapAny, sinkSchema) - val jsonScenario = createEditorModeScenario(cfg, sinkFields) + val cfg = config(dummyInputObject, schemaMapAny, sinkSchema) + val jsonScenario = createEditorModeScenario(cfg, sinkFields) runner.registerJsonSchema(cfg.sourceTopic.toUnspecialized, cfg.sourceSchema) runner.registerJsonSchema(cfg.sinkTopic.toUnspecialized, cfg.sinkSchema) @@ -410,7 +413,7 @@ class LiteKafkaUniversalJsonFunctionalTest test("various combinations of optional-like fields with sink in editor mode") { def expectValidObject(expectedObject: Map[String, Json])( - result: ValidatedNel[ProcessCompilationError, Map[String, Json]] + result: ValidatedNel[ProcessCompilationError, Map[String, Json]] ): Assertion = result shouldBe Valid(expectedObject) @@ -444,8 +447,8 @@ class LiteKafkaUniversalJsonFunctionalTest ) ) { (outputSchema, fieldExpression, expectationCheckingFun) => val dummyInputObject = obj() - val cfg = config(dummyInputObject, schemaMapAny, outputSchema) - val jsonScenario = createEditorModeScenario(cfg, Map(ObjectFieldName -> fieldExpression)) + val cfg = config(dummyInputObject, schemaMapAny, outputSchema) + val jsonScenario = createEditorModeScenario(cfg, Map(ObjectFieldName -> fieldExpression)) runner.registerJsonSchema(cfg.sourceTopic.toUnspecialized, cfg.sourceSchema) runner.registerJsonSchema(cfg.sinkTopic.toUnspecialized, cfg.sinkSchema) val input = KafkaConsumerRecord[String, String](cfg.sourceTopic, cfg.inputData.toString()) @@ -529,7 +532,7 @@ class LiteKafkaUniversalJsonFunctionalTest val strictValidationErrors = runWithValueResults(strictConfig).invalidValue strictValidationErrors should matchPattern { case NonEmptyList(CustomNodeError(`sinkName`, message, _), Nil) - if message.contains(s"Redundant fields: $secondsField") => + if message.contains(s"Redundant fields: $secondsField") => } val laxConfig = strictConfig.copy(validationMode = Some(ValidationMode.lax)) @@ -540,13 +543,13 @@ class LiteKafkaUniversalJsonFunctionalTest } private def createEditorModeScenario( - config: ScenarioConfig, - fieldsExpressions: Map[String, String] - ): CanonicalProcess = { + config: ScenarioConfig, + fieldsExpressions: Map[String, String] + ): CanonicalProcess = { val sinkParams = (Map( - topicParamName.value -> s"'${config.sinkTopic.name}'", + topicParamName.value -> s"'${config.sinkTopic.name}'", schemaVersionParamName.value -> s"'${SchemaVersionOption.LatestOptionName}'", - sinkKeyParamName.value -> "", + sinkKeyParamName.value -> "", sinkRawEditorParamName.value -> "false", ) ++ fieldsExpressions).mapValuesNow(Expression.spel) @@ -555,7 +558,7 @@ class LiteKafkaUniversalJsonFunctionalTest .source( sourceName, KafkaUniversalName, - topicParamName.value -> s"'${config.sourceTopic.name}'".spel, + topicParamName.value -> s"'${config.sourceTopic.name}'".spel, schemaVersionParamName.value -> s"'${SchemaVersionOption.LatestOptionName}'".spel ) .emptySink(sinkName, KafkaUniversalName, sinkParams.toList: _*) @@ -591,7 +594,7 @@ class LiteKafkaUniversalJsonFunctionalTest forAll(testData) { (input: Json, sourceSchema: EveritSchema, expected: String) => // here we're testing only source runtime validation so to prevent typing issues we pass literal as sink expression - val cfg = config(input, sourceSchema, schemaString, output = "someString") + val cfg = config(input, sourceSchema, schemaString, output = "someString") val results = runWithValueResults(cfg) val message = results.validValue.errors.head.throwable.asInstanceOf[RuntimeException].getMessage @@ -645,7 +648,7 @@ class LiteKafkaUniversalJsonFunctionalTest runner.registerJsonSchema(config.sourceTopic.toUnspecialized, config.sourceSchema) runner.registerJsonSchema(config.sinkTopic.toUnspecialized, config.sinkSchema) - val input = KafkaConsumerRecord[String, String](config.sourceTopic, config.inputData.toString()) + val input = KafkaConsumerRecord[String, String](config.sourceTopic, config.inputData.toString()) val result = runner.runWithStringData(jsonScenario, List(input)) result } @@ -656,47 +659,47 @@ class LiteKafkaUniversalJsonFunctionalTest .source( sourceName, KafkaUniversalName, - topicParamName.value -> s"'${config.sourceTopic.name}'".spel, + topicParamName.value -> s"'${config.sourceTopic.name}'".spel, schemaVersionParamName.value -> s"'${SchemaVersionOption.LatestOptionName}'".spel ) .emptySink( sinkName, KafkaUniversalName, - topicParamName.value -> s"'${config.sinkTopic.name}'".spel, - schemaVersionParamName.value -> s"'${SchemaVersionOption.LatestOptionName}'".spel, - sinkKeyParamName.value -> "".spel, - sinkValueParamName.value -> s"${config.sinkDefinition}".spel, - sinkRawEditorParamName.value -> "true".spel, + topicParamName.value -> s"'${config.sinkTopic.name}'".spel, + schemaVersionParamName.value -> s"'${SchemaVersionOption.LatestOptionName}'".spel, + sinkKeyParamName.value -> "".spel, + sinkValueParamName.value -> s"${config.sinkDefinition}".spel, + sinkRawEditorParamName.value -> "true".spel, sinkValidationModeParamName.value -> s"'${config.validationModeName}'".spel ) case class ScenarioConfig( - topic: String, - inputData: Json, - sourceSchema: EveritSchema, - sinkSchema: EveritSchema, - sinkDefinition: String, - validationMode: Option[ValidationMode] - ) { + topic: String, + inputData: Json, + sourceSchema: EveritSchema, + sinkSchema: EveritSchema, + sinkDefinition: String, + validationMode: Option[ValidationMode] + ) { lazy val validationModeName: String = validationMode.map(_.name).getOrElse(ValidationMode.strict.name) - lazy val sourceTopic = TopicName.ForSource(s"$topic-input") - lazy val sinkTopic = TopicName.ForSink(s"$topic-output") + lazy val sourceTopic = TopicName.ForSource(s"$topic-input") + lazy val sinkTopic = TopicName.ForSink(s"$topic-output") } private def conf( - outputSchema: EveritSchema, - output: Any = Input, - validationMode: Option[ValidationMode] = None - ): ScenarioConfig = + outputSchema: EveritSchema, + output: Any = Input, + validationMode: Option[ValidationMode] = None + ): ScenarioConfig = config(Null, schemaNull, outputSchema, output, validationMode) private def config( - inputData: Json, - sourceSchema: EveritSchema, - sinkSchema: EveritSchema, - output: Any = Input, - validationMode: Option[ValidationMode] = None - ): ScenarioConfig = + inputData: Json, + sourceSchema: EveritSchema, + sinkSchema: EveritSchema, + output: Any = Input, + validationMode: Option[ValidationMode] = None + ): ScenarioConfig = ScenarioConfig(randomTopic, inputData, sourceSchema, sinkSchema, output.toSpELLiteral, validationMode) } diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/spel/Typer.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/spel/Typer.scala index a9a10911c70..f8744e0555b 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/spel/Typer.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/spel/Typer.scala @@ -16,6 +16,7 @@ import pl.touk.nussknacker.engine.api.context.ValidationContext import pl.touk.nussknacker.engine.api.expression._ import pl.touk.nussknacker.engine.api.generics.ExpressionParseError import pl.touk.nussknacker.engine.api.typed.supertype.{CommonSupertypeFinder, NumberTypesPromotionStrategy} +import pl.touk.nussknacker.engine.api.typed.typing.Typed.typedListWithElementValues import pl.touk.nussknacker.engine.api.typed.typing._ import pl.touk.nussknacker.engine.definition.clazz.ClassDefinitionSet import pl.touk.nussknacker.engine.definition.globalvariables.ExpressionConfigDefinition @@ -52,6 +53,7 @@ import pl.touk.nussknacker.engine.spel.internal.EvaluationContextPreparer import pl.touk.nussknacker.engine.spel.typer.{MapLikePropertyTyper, MethodReferenceTyper, TypeReferenceTyper} import pl.touk.nussknacker.engine.util.MathUtils +import scala.jdk.CollectionConverters._ import scala.annotation.tailrec import scala.reflect.runtime._ import scala.util.{Failure, Success, Try} @@ -225,6 +227,7 @@ private[spel] class Typer( // TODO: how to handle other cases? case TypedNull => invalidNodeResult(IllegalIndexingOperation) + case TypedObjectWithValue(underlying, _) => typeIndexer(e, underlying) case _ => val w = validNodeResult(Unknown) if (dynamicPropertyAccessAllowed) w else w.tell(List(DynamicPropertyAccessError)) @@ -302,8 +305,10 @@ private[spel] class Typer( def getSupertype(a: TypingResult, b: TypingResult): TypingResult = CommonSupertypeFinder.Default.commonSupertype(a, b) - val elementType = if (children.isEmpty) Unknown else children.reduce(getSupertype) - valid(Typed.genericTypeClass[java.util.List[_]](List(elementType))) + val elementType = if (children.isEmpty) Unknown else children.reduce(getSupertype).withoutValue + val childrenCombinedValue = children.flatMap(_.valueOpt).asJava + + valid(typedListWithElementValues(elementType, childrenCombinedValue)) } case e: InlineMap => @@ -418,6 +423,8 @@ private[spel] class Typer( elementType <- extractIterativeType(iterateType) result <- typeChildren(validationContext, node, current.pushOnStack(elementType)) { case result :: Nil => + // Limitation: projection on an iterative type makes it loses it's known value, + // as properly determining it would require evaluating the projection expression for each element (likely working on the AST) valid(Typed.genericTypeClass[java.util.List[_]](List(result))) case other => invalid(IllegalSelectionTypeError(other)) @@ -502,7 +509,23 @@ private[spel] class Typer( childElementType: TypingResult ) = { val isSingleElementSelection = List("$", "^").map(node.toStringAST.startsWith(_)).foldLeft(false)(_ || _) - if (isSingleElementSelection) childElementType else parentType + + if (isSingleElementSelection) + childElementType + else { + // Limitation: selection from an iterative type makes it loses it's known value, + // as properly determining it would require evaluating the selection expression for each element (likely working on the AST) + parentType match { + case tc: SingleTypingResult if tc.objType.canBeSubclassOf(Typed[java.util.Collection[_]]) => + tc.withoutValue + case tc: SingleTypingResult if tc.objType.klass.isArray => + tc.withoutValue + case tc: SingleTypingResult if tc.objType.canBeSubclassOf(Typed[java.util.Map[_, _]]) => + Typed.record(Map.empty) + case _ => + parentType + } + } } private def isArrayConstructor(constructorReference: ConstructorReference): Boolean = { diff --git a/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/compile/GenericTransformationValidationSpec.scala b/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/compile/GenericTransformationValidationSpec.scala index 6f22cb80d8b..5c30b667060 100644 --- a/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/compile/GenericTransformationValidationSpec.scala +++ b/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/compile/GenericTransformationValidationSpec.scala @@ -27,6 +27,8 @@ import pl.touk.nussknacker.engine.dict.SimpleDictRegistry import pl.touk.nussknacker.engine.modelconfig.ComponentsUiConfig import pl.touk.nussknacker.engine.testing.ModelDefinitionBuilder import pl.touk.nussknacker.engine.CustomProcessValidatorLoader +import pl.touk.nussknacker.engine.api.typed.typing.Typed.typedListWithElementValues +import scala.jdk.CollectionConverters._ class GenericTransformationValidationSpec extends AnyFunSuite with Matchers with OptionValues with Inside { @@ -98,7 +100,7 @@ class GenericTransformationValidationSpec extends AnyFunSuite with Matchers with Map( "val1" -> Typed.fromInstance("aa"), "val2" -> Typed.fromInstance(11), - "val3" -> Typed.genericTypeClass(classOf[java.util.List[_]], List(Typed.fromInstance(false))) + "val3" -> typedListWithElementValues(Typed[java.lang.Boolean], List(false).asJava) ) ) @@ -127,7 +129,7 @@ class GenericTransformationValidationSpec extends AnyFunSuite with Matchers with Map( "val1" -> Typed.fromInstance("aa"), "val2" -> Typed.fromInstance(11), - "val3" -> Typed.genericTypeClass(classOf[java.util.List[_]], List(Typed.fromInstance(false))) + "val3" -> typedListWithElementValues(Typed[java.lang.Boolean], List(false).asJava) ) ) diff --git a/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/spel/SpelExpressionSpec.scala b/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/spel/SpelExpressionSpec.scala index 7f11ecd861c..60264491378 100644 --- a/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/spel/SpelExpressionSpec.scala +++ b/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/spel/SpelExpressionSpec.scala @@ -22,6 +22,7 @@ import pl.touk.nussknacker.engine.api.generics.{ } import pl.touk.nussknacker.engine.api.process.ExpressionConfig._ import pl.touk.nussknacker.engine.api.typed.TypedMap +import pl.touk.nussknacker.engine.api.typed.typing.Typed.typedListWithElementValues import pl.touk.nussknacker.engine.api.typed.typing.{Typed, _} import pl.touk.nussknacker.engine.api.{Context, NodeId, SpelExpressionExcludeList} import pl.touk.nussknacker.engine.definition.clazz.{ClassDefinitionSet, JavaClassWithVarargs} @@ -351,10 +352,9 @@ class SpelExpressionSpec extends AnyFunSuite with Matchers with ValidatedValuesD parse[Any]("null").toOption.get.returnType shouldBe TypedNull parse[java.util.List[String]]("{'t', null, 'a'}").toOption.get.returnType shouldBe - Typed.genericTypeClass(classOf[java.util.List[_]], List(Typed[String])) + typedListWithElementValues(Typed[String], List("t", null, "a").asJava) parse[java.util.List[Any]]("{5, 't', null}").toOption.get.returnType shouldBe - Typed.genericTypeClass(classOf[java.util.List[_]], List(Typed[Any])) - + typedListWithElementValues(Typed[Any], List(5, "t", null).asJava) parse[Int]("true ? 8 : null").toOption.get.returnType shouldBe Typed[Int] } @@ -816,7 +816,7 @@ class SpelExpressionSpec extends AnyFunSuite with Matchers with ValidatedValuesD ) shouldHaveBadType( parse[String]("{1, 2, 3}", ctx), - s"Bad expression type, expected: String, found: ${Typed.genericTypeClass(classOf[java.util.List[_]], List(Typed.typedClass[Int])).display}" + s"Bad expression type, expected: String, found: ${typedListWithElementValues(Typed[Int], List(1, 2, 3).asJava).display}" ) shouldHaveBadType( parse[java.util.Map[_, _]]("'alaMa'", ctx), diff --git a/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/spel/TyperSpec.scala b/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/spel/TyperSpec.scala index 33a529ba72b..fa73148d935 100644 --- a/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/spel/TyperSpec.scala +++ b/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/spel/TyperSpec.scala @@ -8,6 +8,7 @@ import org.springframework.expression.common.TemplateParserContext import org.springframework.expression.spel.standard import pl.touk.nussknacker.engine.api.context.ValidationContext import pl.touk.nussknacker.engine.api.generics.ExpressionParseError +import pl.touk.nussknacker.engine.api.typed.typing.Typed.typedListWithElementValues import pl.touk.nussknacker.engine.api.typed.typing._ import pl.touk.nussknacker.engine.definition.clazz.ClassDefinitionSet import pl.touk.nussknacker.engine.dict.{KeysDictTyper, SimpleDictRegistry} @@ -19,6 +20,8 @@ import pl.touk.nussknacker.engine.spel.TyperSpecTestData.TestRecord._ import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap import pl.touk.nussknacker.test.ValidatedValuesDetailedMessage +import scala.jdk.CollectionConverters._ + class TyperSpec extends AnyFunSuite with Matchers with ValidatedValuesDetailedMessage { private implicit val defaultTyper: Typer = buildTyper() @@ -45,9 +48,28 @@ class TyperSpec extends AnyFunSuite with Matchers with ValidatedValuesDetailedMe ) } - test("detect proper selection types") { + test("detect proper List type with value - record inside") { + typeExpression(s"{$testRecordExpr}").validValue.finalResult.typingResult shouldBe + typedListWithElementValues( + testRecordTyped.withoutValue, + List(testRecordTyped.valueOpt.get).asJava + ) + } + + test("detect proper List type with value") { + typeExpression("{1,2}").validValue.finalResult.typingResult shouldBe + typedListWithElementValues(Typed.typedClass[Int], List(1, 2).asJava) + } + + test("detect proper selection types - List") { typeExpression("{1,2}.?[(#this==1)]").validValue.finalResult.typingResult shouldBe Typed.genericTypeClass(classOf[java.util.List[_]], List(Typed.typedClass[Int])) + // see comment in Typer.resolveSelectionTypingResult + } + + test("detect proper selection types - Map") { + typeExpression("{'field1': 1, 'field2': 2}.?[(#this.value==1)]").validValue.finalResult.typingResult shouldBe + Typed.record(Map.empty) // see comment in Typer.resolveSelectionTypingResult } test("detect proper first selection types") { @@ -73,6 +95,11 @@ class TyperSpec extends AnyFunSuite with Matchers with ValidatedValuesDetailedMe s"Cannot do projection/selection on ${Typed.fromInstance(1).display}" } + test("type record expression") { + typeExpression(testRecordExpr).validValue.finalResult.typingResult shouldBe + testRecordTyped + } + test("indexing on records for primitive types") { typeExpression(s"$testRecordExpr['string']").validValue.finalResult.typingResult shouldBe TypedObjectWithValue(Typed.typedClass[String], "stringVal") diff --git a/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/collection.scala b/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/collection.scala index e7d179b2edb..d8eb6f7299d 100644 --- a/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/collection.scala +++ b/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/collection.scala @@ -6,7 +6,13 @@ import org.springframework.util.{NumberUtils => SpringNumberUtils} import pl.touk.nussknacker.engine.api.generics.{GenericFunctionTypingError, GenericType, TypingFunction} import pl.touk.nussknacker.engine.api.typed.supertype.NumberTypesPromotionStrategy.ForLargeNumbersOperation import pl.touk.nussknacker.engine.api.typed.typing -import pl.touk.nussknacker.engine.api.typed.typing.{Typed, TypedClass, TypedObjectTypingResult, Unknown} +import pl.touk.nussknacker.engine.api.typed.typing.{ + Typed, + TypedClass, + TypedObjectTypingResult, + TypedObjectWithValue, + Unknown +} import pl.touk.nussknacker.engine.api.{Documentation, HideToString, ParamName} import java.util.{Collections, Objects} @@ -204,8 +210,10 @@ object CollectionUtils { arguments: List[typing.TypingResult] ): ValidatedNel[GenericFunctionTypingError, typing.TypingResult] = arguments match { case (f @ TypedClass(`fClass`, element :: Nil)) :: _ => f.copy(params = element.withoutValue :: Nil).validNel - case firstArgument :: _ => firstArgument.validNel - case _ => GenericFunctionTypingError.ArgumentTypeError.invalidNel + case TypedObjectWithValue(f @ TypedClass(`fClass`, element :: Nil), _) :: _ => + f.copy(params = element.withoutValue :: Nil).validNel + case firstArgument :: _ => firstArgument.validNel + case _ => GenericFunctionTypingError.ArgumentTypeError.invalidNel } } @@ -217,8 +225,10 @@ object CollectionUtils { arguments: List[typing.TypingResult] ): ValidatedNel[GenericFunctionTypingError, typing.TypingResult] = arguments match { case TypedClass(`fClass`, componentType :: Nil) :: _ => componentType.withoutValue.validNel - case firstArgument :: _ => firstArgument.withoutValue.validNel - case _ => GenericFunctionTypingError.ArgumentTypeError.invalidNel + case TypedObjectWithValue(TypedClass(`fClass`, componentType :: Nil), _) :: _ => + componentType.withoutValue.validNel + case firstArgument :: _ => firstArgument.withoutValue.validNel + case _ => GenericFunctionTypingError.ArgumentTypeError.invalidNel } } @@ -259,10 +269,30 @@ object CollectionUtils { override def computeResultType( arguments: List[typing.TypingResult] ): ValidatedNel[GenericFunctionTypingError, typing.TypingResult] = arguments match { - case (listType @ TypedClass(`fClass`, firstComponentType :: Nil)) :: TypedClass( - `fClass`, - secondComponentType :: Nil - ) :: Nil => + case (list1 @ TypedClass(`fClass`, _ :: Nil)) :: + (list2 @ TypedClass(`fClass`, _ :: Nil)) :: Nil => + concatType(list1, list2) + + case (list1 @ TypedClass(`fClass`, _ :: Nil)) :: + TypedObjectWithValue(list2 @ TypedClass(`fClass`, _ :: Nil), _) :: Nil => + concatType(list1, list2) + + case TypedObjectWithValue(list1 @ TypedClass(`fClass`, _ :: Nil), _) :: + (list2 @ TypedClass(`fClass`, _ :: Nil)) :: Nil => + concatType(list1, list2) + + case TypedObjectWithValue(list1 @ TypedClass(`fClass`, _ :: Nil), _) :: + TypedObjectWithValue(list2 @ TypedClass(`fClass`, _ :: Nil), _) :: Nil => + concatType(list1, list2) + + case _ => Typed.genericTypeClass(fClass, List(Unknown)).validNel + } + + private def concatType(list1: TypedClass, list2: TypedClass) = (list1, list2) match { + case ( + listType @ TypedClass(`fClass`, firstComponentType :: Nil), + TypedClass(`fClass`, secondComponentType :: Nil) + ) => (firstComponentType, secondComponentType) match { case (TypedObjectTypingResult(x, _, infoX), TypedObjectTypingResult(y, _, infoY)) if commonFieldHasTheSameType(x, y) => diff --git a/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/BaseSpelSpec.scala b/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/BaseSpelSpec.scala index c2a89af3529..ac393226cd1 100644 --- a/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/BaseSpelSpec.scala +++ b/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/BaseSpelSpec.scala @@ -7,7 +7,7 @@ import pl.touk.nussknacker.engine.api.typed.typing.{Typed, TypingResult, Unknown import pl.touk.nussknacker.engine.api.Context import pl.touk.nussknacker.engine.definition.clazz.ClassDefinitionSet import pl.touk.nussknacker.engine.dict.SimpleDictRegistry -import pl.touk.nussknacker.engine.expression.parse.CompiledExpression +import pl.touk.nussknacker.engine.expression.parse.{CompiledExpression, TypedExpression} import pl.touk.nussknacker.engine.spel.SpelExpressionParser import pl.touk.nussknacker.engine.testing.ModelDefinitionBuilder import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap diff --git a/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/CollectionUtilsSpec.scala b/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/CollectionUtilsSpec.scala index 7c9588ed434..e9feb0c2e7b 100644 --- a/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/CollectionUtilsSpec.scala +++ b/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/CollectionUtilsSpec.scala @@ -63,7 +63,7 @@ class CollectionUtilsSpec extends AnyFunSuite with BaseSpelSpec with Matchers { ), ( "#COLLECTION.merge({a:{innerA:{10,20}}},{b:{innerB:{20}}})", - "Record{a: Record{innerA: List[Integer]}, b: Record{innerB: List[Integer(20)]}}" + "Record{a: Record{innerA: List[Integer]({10, 20})}, b: Record{innerB: List[Integer]({20})}}" ), ("#COLLECTION.merge({a:4,b:3},{a:'5'})", "Record{a: String(5), b: Integer(3)}"), ("#COLLECTION.merge(#unknownMap,{a:'5'})", "Map[Unknown,Unknown]"), @@ -316,7 +316,7 @@ class CollectionUtilsSpec extends AnyFunSuite with BaseSpelSpec with Matchers { evaluateType("#COLLECTION.flatten({{'1'},{'2', '3'},{'3'}})") shouldBe "List[String]".valid evaluateType("#COLLECTION.flatten({{1},{2},{3}})") shouldBe "List[Integer]".valid evaluateType("#COLLECTION.flatten({{{a:1},{b:2}},{{c:3},{d:4}}})") shouldBe - "List[Record{a: Integer(1), b: Integer(2), c: Integer(3), d: Integer(4)}]".valid + "List[Record{a: Integer, b: Integer, c: Integer, d: Integer}]".valid } test("should throw if elements are not comparable") { @@ -335,7 +335,7 @@ class CollectionUtilsSpec extends AnyFunSuite with BaseSpelSpec with Matchers { evaluateType(expression, variables) } caught.getMessage should include( - "NonComparable that does not match any of declared types (Comparable[Unknown]) when called with arguments (List[NonComparable])" + "NonComparable that does not match any of declared types (Comparable[Unknown]) when called with arguments (List[NonComparable]" ) } diff --git a/utils/math-utils/src/main/scala/pl/touk/nussknacker/engine/util/MathUtils.scala b/utils/math-utils/src/main/scala/pl/touk/nussknacker/engine/util/MathUtils.scala index 41f83d76945..2455028ad53 100644 --- a/utils/math-utils/src/main/scala/pl/touk/nussknacker/engine/util/MathUtils.scala +++ b/utils/math-utils/src/main/scala/pl/touk/nussknacker/engine/util/MathUtils.scala @@ -8,6 +8,7 @@ import pl.touk.nussknacker.engine.api.typed.supertype.{ } import java.lang +import java.math.RoundingMode trait MathUtils { @@ -114,8 +115,13 @@ trait MathUtils { n1.divide(n2) override def onFloats(n1: java.lang.Float, n2: java.lang.Float): java.lang.Float = n1 / n2 override def onDoubles(n1: java.lang.Double, n2: java.lang.Double): java.lang.Double = n1 / n2 - override def onBigDecimals(n1: java.math.BigDecimal, n2: java.math.BigDecimal): java.math.BigDecimal = - n1.divide(n2) + override def onBigDecimals(n1: java.math.BigDecimal, n2: java.math.BigDecimal): java.math.BigDecimal = { + n1.divide( + n2, + Math.max(n1.scale(), n2.scale), + RoundingMode.HALF_EVEN + ) // same scale and rounding as used by OpDivide in SpelExpression.java + } })(NumberTypesPromotionStrategy.ForMathOperation) } diff --git a/utils/schemed-kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/schemedkafka/encode/AvroSchemaOutputValidator.scala b/utils/schemed-kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/schemedkafka/encode/AvroSchemaOutputValidator.scala index fd978680f94..28c5c37d8f6 100644 --- a/utils/schemed-kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/schemedkafka/encode/AvroSchemaOutputValidator.scala +++ b/utils/schemed-kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/schemedkafka/encode/AvroSchemaOutputValidator.scala @@ -62,6 +62,9 @@ class AvroSchemaOutputValidator(validationMode: ValidationMode) extends LazyLogg validateMapSchema(typingResult, schema, path) case (tc @ TypedClass(cl, _), Type.ARRAY) if classOf[java.util.List[_]].isAssignableFrom(cl) => validateArraySchema(tc, schema, path) + case (TypedObjectWithValue(tc @ TypedClass(cl, _), _), Type.ARRAY) + if classOf[java.util.List[_]].isAssignableFrom(cl) => + validateArraySchema(tc, schema, path) case (TypedNull, _) if !schema.isNullable => invalid(typingResult, schema, path) case (TypedNull, _) if schema.isNullable =>