From a1344c0545d72d540c1b408a902cee5b42a2101b Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Tue, 30 Apr 2024 16:31:44 +0100 Subject: [PATCH 01/19] #1318 example of how to use schema mapper to get data and update in memory table --- .../vuu/example/ignite/SchemaExample.scala | 176 ++++++++++++++++++ .../org/finos/vuu/api/ColumnBuilder.scala | 32 ++++ 2 files changed, 208 insertions(+) create mode 100644 example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/SchemaExample.scala create mode 100644 vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala diff --git a/example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/SchemaExample.scala b/example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/SchemaExample.scala new file mode 100644 index 000000000..53891c825 --- /dev/null +++ b/example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/SchemaExample.scala @@ -0,0 +1,176 @@ +package org.finos.vuu.example.ignite + +import org.finos.vuu.api.{ColumnBuilder, TableDef} +import org.finos.vuu.core.index.IndexedField +import org.finos.vuu.core.table.{Column, ColumnValueProvider, DataTable, KeyObserver, RowData, RowKeyUpdate, RowWithData, TableData, TablePrimaryKeys} +import org.finos.vuu.example.ignite.utils.getListToObjectConverter +import org.finos.vuu.util.schema.{ExternalEntitySchema, ExternalEntitySchemaBuilder, SchemaMapperBuilder} +import org.finos.vuu.viewport.{RowProcessor, ViewPortColumns} + +class SchemaExample { + + //todo - try in java + //try more scenarios + // - virtual table - filter, type ahead + // - error scenarios fetching data or converting types + //error handling - review api for returning error + + //create table def (use column builder) + val tableDef = TableDef( + name = "MyExampleTable", + keyField = "Id", + columns = new ColumnBuilder() + .addString("Id") + .addDouble("NotionalValue") + .build + ) + + //create entity schema + val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() + .withEntity(classOf[SchemaTestData]) + .withIndex("ID_INDEX", List("id")) + .build() + + //create schema mapper + private val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + //.withFieldsMap(columnNameByExternalField) + .build() + + //get data from ignite as list of values + private val queryName = "myQuery" + val igniteStore: FakeIgniteStore = new FakeIgniteStore + igniteStore.setUpSqlFieldsQuery( + queryName, + List[List[Any]]( + "id1", 10.5 + ) + ) + + //this is one row - should be multiple? + val result = igniteStore.getSqlFieldsQuery(queryName) + .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) + + // map to entity object - as order of values are relevant to how the query schema was defined + + val tableRowMap1 = result + .map(rowData => mapToEntity(rowData)) + .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)) + + private val tableRowMap2: Seq[Map[String, Any]] = result.map(rowData => schemaMapper.toInternalRowMap(rowData)) + + //map to tablerow + val keyFieldName = tableDef.keyField + val tableRows = tableRowMap2.map(rowMap => + { + val keyValue = rowMap(keyFieldName).toString + RowWithData(keyValue, rowMap) + }) + + //update table with table row? + + var table = new FakeInMemoryTable("SchemaMapTest", tableDef) + tableRows.foreach(row => table.processUpdate(row.key, row)) + + + //assert on reading the table row - is that possible or need to use mock table with table interface + var existingRows = table.pullAllRows() + + //todo different for java + private def mapToEntity(rowData: List[Any]): SchemaTestData = + getListToObjectConverter[SchemaTestData](SchemaTestData)(rowData) +} + +class FakeIgniteStore(){ + + private val queryToEntityResultMap = scala.collection.mutable.HashMap.empty[String, List[SchemaTestData]] + private val queryToValuesResultMap = scala.collection.mutable.HashMap.empty[String, List[List[Any]]] + + def setUpIndexQuery(queryName:String, queryResult:List[SchemaTestData]): Unit = { + queryToEntityResultMap += (queryName -> queryResult) + } + + def getIndexQuery(queryName: String): Option[List[SchemaTestData]] = { + queryToEntityResultMap.get(queryName) + } + + def setUpSqlFieldsQuery(queryName: String, resultValues: List[List[Any]]): Unit = { + queryToValuesResultMap += (queryName -> resultValues) + } + + def getSqlFieldsQuery(queryName: String): Option[List[List[Any]]] = { + queryToValuesResultMap.get(queryName) + } +} + +case class SchemaTestData(id :String, notionalValue: Double) { +} + +class FakeInMemoryTable(name:String, tableDef: TableDef) extends DataTable { + + private val rowMap = scala.collection.mutable.HashMap.empty[String, RowWithData] + + override def name: String = name + override def getTableDef: TableDef = tableDef + + override def processUpdate(rowKey: String, rowUpdate: RowWithData, timeStamp: Long = 0): Unit = + rowMap += (rowKey -> rowUpdate) + + override def pullRow(key: String): RowData = + rowMap.get(key) + .getOrElse(throw new Exception(s"Could not find row data for key $key in table $name")) + + def pullAllRows() : List[RowWithData] = rowMap.values.toList + + override protected def createDataTableData(): TableData = ??? + + override def updateCounter: Long = ??? + + override def incrementUpdateCounter(): Unit = ??? + + override def indexForColumn(column: Column): Option[IndexedField[_]] = ??? + + override def getColumnValueProvider: ColumnValueProvider = ??? + + override def processDelete(rowKey: String): Unit = ??? + + /** + * notify listeners explicit when a rowKey changes + */ + override def notifyListeners(rowKey: String, isDelete: Boolean): Unit = ??? + + /** + * Link table name is the name of the underlying table that we can link to. + * In a session table this would be the underlying table. + * + * @return + */ + override def linkableName: String = ??? + + override def readRow(key: String, columns: List[String], processor: RowProcessor): Unit = ??? + + override def primaryKeys: TablePrimaryKeys = ??? + + override def pullRow(key: String, columns: ViewPortColumns): RowData = ??? + + /** + * Note the below call should only be used for testing. It filters the contents of maps by the expected viewPortColumns. + * In practice we never need to do this at runtime. + */ + override def pullRowFiltered(key: String, columns: ViewPortColumns): RowData = ??? + + override def pullRowAsArray(key: String, columns: ViewPortColumns): Array[Any] = ??? + + override def getObserversByKey(): Map[String, Array[KeyObserver[RowKeyUpdate]]] = ??? + + override def addKeyObserver(key: String, observer: KeyObserver[RowKeyUpdate]): Boolean = ??? + + override def removeKeyObserver(key: String, observer: KeyObserver[RowKeyUpdate]): Boolean = ??? + + override def getObserversByKey(key: String): List[KeyObserver[RowKeyUpdate]] = ??? + + override def isKeyObserved(key: String): Boolean = ??? + + override def isKeyObservedBy(key: String, observer: KeyObserver[RowKeyUpdate]): Boolean = ??? + + override def removeAllObservers(): Unit = ??? +} diff --git a/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala new file mode 100644 index 000000000..870800258 --- /dev/null +++ b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala @@ -0,0 +1,32 @@ +package org.finos.vuu.api + +import org.finos.vuu.core.table.{Column, Columns} + +import scala.collection.mutable.ArrayBuilder + +class ColumnBuilder { + + val columns = new ArrayBuilder.ofRef[String]() + + def addString(columnName: String): ColumnBuilder = { + columns +: (columnName + ":String") + this + } + + def addDouble(columnName: String): ColumnBuilder = { + columns +: (columnName + ":Double") + this + } + + def addInt(columnName: String): ColumnBuilder = { + columns +: (columnName + ":Int") + this + } + + def addLong(columnName: String): ColumnBuilder = { + columns +: (columnName + ":Long") + this + } + + def build: Array[Column] = Columns.fromNames(columns.result()) +} From 9ea52fb5c5974ab76816353776aa22535b0be15f Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Tue, 30 Apr 2024 16:32:04 +0100 Subject: [PATCH 02/19] #1318 helper function for scala and java collection converter --- .../vuu/example/ignite/SchemaExample.scala | 176 ------------------ .../vuu/util/ScalaCollectionConverter.java | 29 +++ .../org/finos/vuu/test/FakeIgniteStore.scala | 25 +++ .../finos/vuu/test/FakeInMemoryTable.scala | 76 ++++++++ .../org/finos/vuu/util/SchemaExample.scala | 87 +++++++++ 5 files changed, 217 insertions(+), 176 deletions(-) delete mode 100644 example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/SchemaExample.scala create mode 100644 vuu/src/main/java/org/finos/vuu/util/ScalaCollectionConverter.java create mode 100644 vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala create mode 100644 vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala create mode 100644 vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala diff --git a/example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/SchemaExample.scala b/example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/SchemaExample.scala deleted file mode 100644 index 53891c825..000000000 --- a/example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/SchemaExample.scala +++ /dev/null @@ -1,176 +0,0 @@ -package org.finos.vuu.example.ignite - -import org.finos.vuu.api.{ColumnBuilder, TableDef} -import org.finos.vuu.core.index.IndexedField -import org.finos.vuu.core.table.{Column, ColumnValueProvider, DataTable, KeyObserver, RowData, RowKeyUpdate, RowWithData, TableData, TablePrimaryKeys} -import org.finos.vuu.example.ignite.utils.getListToObjectConverter -import org.finos.vuu.util.schema.{ExternalEntitySchema, ExternalEntitySchemaBuilder, SchemaMapperBuilder} -import org.finos.vuu.viewport.{RowProcessor, ViewPortColumns} - -class SchemaExample { - - //todo - try in java - //try more scenarios - // - virtual table - filter, type ahead - // - error scenarios fetching data or converting types - //error handling - review api for returning error - - //create table def (use column builder) - val tableDef = TableDef( - name = "MyExampleTable", - keyField = "Id", - columns = new ColumnBuilder() - .addString("Id") - .addDouble("NotionalValue") - .build - ) - - //create entity schema - val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() - .withEntity(classOf[SchemaTestData]) - .withIndex("ID_INDEX", List("id")) - .build() - - //create schema mapper - private val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) - //.withFieldsMap(columnNameByExternalField) - .build() - - //get data from ignite as list of values - private val queryName = "myQuery" - val igniteStore: FakeIgniteStore = new FakeIgniteStore - igniteStore.setUpSqlFieldsQuery( - queryName, - List[List[Any]]( - "id1", 10.5 - ) - ) - - //this is one row - should be multiple? - val result = igniteStore.getSqlFieldsQuery(queryName) - .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) - - // map to entity object - as order of values are relevant to how the query schema was defined - - val tableRowMap1 = result - .map(rowData => mapToEntity(rowData)) - .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)) - - private val tableRowMap2: Seq[Map[String, Any]] = result.map(rowData => schemaMapper.toInternalRowMap(rowData)) - - //map to tablerow - val keyFieldName = tableDef.keyField - val tableRows = tableRowMap2.map(rowMap => - { - val keyValue = rowMap(keyFieldName).toString - RowWithData(keyValue, rowMap) - }) - - //update table with table row? - - var table = new FakeInMemoryTable("SchemaMapTest", tableDef) - tableRows.foreach(row => table.processUpdate(row.key, row)) - - - //assert on reading the table row - is that possible or need to use mock table with table interface - var existingRows = table.pullAllRows() - - //todo different for java - private def mapToEntity(rowData: List[Any]): SchemaTestData = - getListToObjectConverter[SchemaTestData](SchemaTestData)(rowData) -} - -class FakeIgniteStore(){ - - private val queryToEntityResultMap = scala.collection.mutable.HashMap.empty[String, List[SchemaTestData]] - private val queryToValuesResultMap = scala.collection.mutable.HashMap.empty[String, List[List[Any]]] - - def setUpIndexQuery(queryName:String, queryResult:List[SchemaTestData]): Unit = { - queryToEntityResultMap += (queryName -> queryResult) - } - - def getIndexQuery(queryName: String): Option[List[SchemaTestData]] = { - queryToEntityResultMap.get(queryName) - } - - def setUpSqlFieldsQuery(queryName: String, resultValues: List[List[Any]]): Unit = { - queryToValuesResultMap += (queryName -> resultValues) - } - - def getSqlFieldsQuery(queryName: String): Option[List[List[Any]]] = { - queryToValuesResultMap.get(queryName) - } -} - -case class SchemaTestData(id :String, notionalValue: Double) { -} - -class FakeInMemoryTable(name:String, tableDef: TableDef) extends DataTable { - - private val rowMap = scala.collection.mutable.HashMap.empty[String, RowWithData] - - override def name: String = name - override def getTableDef: TableDef = tableDef - - override def processUpdate(rowKey: String, rowUpdate: RowWithData, timeStamp: Long = 0): Unit = - rowMap += (rowKey -> rowUpdate) - - override def pullRow(key: String): RowData = - rowMap.get(key) - .getOrElse(throw new Exception(s"Could not find row data for key $key in table $name")) - - def pullAllRows() : List[RowWithData] = rowMap.values.toList - - override protected def createDataTableData(): TableData = ??? - - override def updateCounter: Long = ??? - - override def incrementUpdateCounter(): Unit = ??? - - override def indexForColumn(column: Column): Option[IndexedField[_]] = ??? - - override def getColumnValueProvider: ColumnValueProvider = ??? - - override def processDelete(rowKey: String): Unit = ??? - - /** - * notify listeners explicit when a rowKey changes - */ - override def notifyListeners(rowKey: String, isDelete: Boolean): Unit = ??? - - /** - * Link table name is the name of the underlying table that we can link to. - * In a session table this would be the underlying table. - * - * @return - */ - override def linkableName: String = ??? - - override def readRow(key: String, columns: List[String], processor: RowProcessor): Unit = ??? - - override def primaryKeys: TablePrimaryKeys = ??? - - override def pullRow(key: String, columns: ViewPortColumns): RowData = ??? - - /** - * Note the below call should only be used for testing. It filters the contents of maps by the expected viewPortColumns. - * In practice we never need to do this at runtime. - */ - override def pullRowFiltered(key: String, columns: ViewPortColumns): RowData = ??? - - override def pullRowAsArray(key: String, columns: ViewPortColumns): Array[Any] = ??? - - override def getObserversByKey(): Map[String, Array[KeyObserver[RowKeyUpdate]]] = ??? - - override def addKeyObserver(key: String, observer: KeyObserver[RowKeyUpdate]): Boolean = ??? - - override def removeKeyObserver(key: String, observer: KeyObserver[RowKeyUpdate]): Boolean = ??? - - override def getObserversByKey(key: String): List[KeyObserver[RowKeyUpdate]] = ??? - - override def isKeyObserved(key: String): Boolean = ??? - - override def isKeyObservedBy(key: String, observer: KeyObserver[RowKeyUpdate]): Boolean = ??? - - override def removeAllObservers(): Unit = ??? -} diff --git a/vuu/src/main/java/org/finos/vuu/util/ScalaCollectionConverter.java b/vuu/src/main/java/org/finos/vuu/util/ScalaCollectionConverter.java new file mode 100644 index 000000000..1987683bf --- /dev/null +++ b/vuu/src/main/java/org/finos/vuu/util/ScalaCollectionConverter.java @@ -0,0 +1,29 @@ +package org.finos.vuu.util; + +import scala.jdk.CollectionConverters; + +import java.util.List; +import java.util.Map; + +public class ScalaCollectionConverter { + + public static scala.collection.immutable.Map toScala(Map m) { + return scala.collection.immutable.Map.from(scala.jdk.CollectionConverters.MapHasAsScala(m).asScala()); + } + + public static scala.collection.Iterable toScala(Iterable l) { + return CollectionConverters.IterableHasAsScala(l).asScala(); + } + + public static scala.collection.immutable.List toScala(List l) { + return CollectionConverters.IterableHasAsScala(l).asScala().toList(); + } + + public static scala.collection.immutable.Seq toScalaSeq(List l) { + return CollectionConverters.IterableHasAsScala(l).asScala().toSeq(); + } + + public static List toJava(scala.collection.immutable.List l) { + return CollectionConverters.SeqHasAsJava(l).asJava(); + } +} diff --git a/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala b/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala new file mode 100644 index 000000000..8b89de14b --- /dev/null +++ b/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala @@ -0,0 +1,25 @@ +package org.finos.vuu.test +class FakeIgniteStore(){ + + private val queryToEntityResultMap = scala.collection.mutable.HashMap.empty[String, List[SchemaTestData]] + private val queryToValuesResultMap = scala.collection.mutable.HashMap.empty[String, List[List[Any]]] + + def setUpIndexQuery(queryName:String, queryResult:List[SchemaTestData]): Unit = { + queryToEntityResultMap += (queryName -> queryResult) + } + + def getIndexQuery(queryName: String): Option[List[SchemaTestData]] = { + queryToEntityResultMap.get(queryName) + } + + def setUpSqlFieldsQuery(queryName: String, resultValues: List[List[Any]]): Unit = { + queryToValuesResultMap += (queryName -> resultValues) + } + + def getSqlFieldsQuery(queryName: String): Option[List[List[Any]]] = { + queryToValuesResultMap.get(queryName) + } +} + +case class SchemaTestData(id :String, notionalValue: Double) { +} \ No newline at end of file diff --git a/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala b/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala new file mode 100644 index 000000000..a4275bc02 --- /dev/null +++ b/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala @@ -0,0 +1,76 @@ +package org.finos.vuu.test + +import org.finos.vuu.api.TableDef +import org.finos.vuu.core.index.IndexedField +import org.finos.vuu.core.table.{Column, ColumnValueProvider, DataTable, KeyObserver, RowData, RowKeyUpdate, RowWithData, TableData, TablePrimaryKeys} +import org.finos.vuu.viewport.{RowProcessor, ViewPortColumns} + +class FakeInMemoryTable(name:String, tableDef: TableDef) extends DataTable { + + private val rowMap = scala.collection.mutable.HashMap.empty[String, RowWithData] + + override def name: String = name + override def getTableDef: TableDef = tableDef + + override def processUpdate(rowKey: String, rowUpdate: RowWithData, timeStamp: Long = 0): Unit = + rowMap += (rowKey -> rowUpdate) + + override def pullRow(key: String): RowData = + rowMap.get(key) + .getOrElse(throw new Exception(s"Could not find row data for key $key in table $name")) + + def pullAllRows() : List[RowWithData] = rowMap.values.toList + + override protected def createDataTableData(): TableData = ??? + + override def updateCounter: Long = ??? + + override def incrementUpdateCounter(): Unit = ??? + + override def indexForColumn(column: Column): Option[IndexedField[_]] = ??? + + override def getColumnValueProvider: ColumnValueProvider = ??? + + override def processDelete(rowKey: String): Unit = ??? + + /** + * notify listeners explicit when a rowKey changes + */ + override def notifyListeners(rowKey: String, isDelete: Boolean): Unit = ??? + + /** + * Link table name is the name of the underlying table that we can link to. + * In a session table this would be the underlying table. + * + * @return + */ + override def linkableName: String = ??? + + override def readRow(key: String, columns: List[String], processor: RowProcessor): Unit = ??? + + override def primaryKeys: TablePrimaryKeys = ??? + + override def pullRow(key: String, columns: ViewPortColumns): RowData = ??? + + /** + * Note the below call should only be used for testing. It filters the contents of maps by the expected viewPortColumns. + * In practice we never need to do this at runtime. + */ + override def pullRowFiltered(key: String, columns: ViewPortColumns): RowData = ??? + + override def pullRowAsArray(key: String, columns: ViewPortColumns): Array[Any] = ??? + + override def getObserversByKey(): Map[String, Array[KeyObserver[RowKeyUpdate]]] = ??? + + override def addKeyObserver(key: String, observer: KeyObserver[RowKeyUpdate]): Boolean = ??? + + override def removeKeyObserver(key: String, observer: KeyObserver[RowKeyUpdate]): Boolean = ??? + + override def getObserversByKey(key: String): List[KeyObserver[RowKeyUpdate]] = ??? + + override def isKeyObserved(key: String): Boolean = ??? + + override def isKeyObservedBy(key: String, observer: KeyObserver[RowKeyUpdate]): Boolean = ??? + + override def removeAllObservers(): Unit = ??? +} \ No newline at end of file diff --git a/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala b/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala new file mode 100644 index 000000000..37e9734ef --- /dev/null +++ b/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala @@ -0,0 +1,87 @@ +package org.finos.vuu.util + +import org.finos.vuu.api.{ColumnBuilder, TableDef} +import org.finos.vuu.core.table.RowWithData +import org.finos.vuu.test.{FakeIgniteStore, FakeInMemoryTable, SchemaTestData} +import org.finos.vuu.util.schema.{ExternalEntitySchema, ExternalEntitySchemaBuilder, SchemaMapperBuilder} + +class SchemaExample { + + //todo - try in java + //try more scenarios + // - virtual table - filter, type ahead + // - error scenarios fetching data or converting types + //error handling - review api for returning error + + //create table def (use column builder) + val tableDef = TableDef( + name = "MyExampleTable", + keyField = "Id", + columns = new ColumnBuilder() + .addString("Id") + .addDouble("NotionalValue") + .build + ) + + //todo to respect the QueryEntity order of fields, should be generated using that? + //create entity schema + val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() + .withEntity(classOf[SchemaTestData]) + .withIndex("ID_INDEX", List("id")) + .build() + + //create schema mapper + private val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + //.withFieldsMap(columnNameByExternalField) + .build() + + //get data from ignite as list of values + private val queryName = "myQuery" + val igniteStore: FakeIgniteStore = new FakeIgniteStore + igniteStore.setUpSqlFieldsQuery( + queryName, + List[List[Any]]( + "id1", 10.5 + ) + ) + + val result = igniteStore.getSqlFieldsQuery(queryName) + .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) + + // map to entity object - as order of values are relevant to how the query schema was defined + val tableRowMap1 = result + .map(rowData => mapToEntity(rowData)) + .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)) + + private val tableRowMap2: Seq[Map[String, Any]] = result.map(rowData => schemaMapper.toInternalRowMap(rowData)) + + //map to tablerow + val keyFieldName = tableDef.keyField + val tableRows = tableRowMap2.map(rowMap => { + val keyValue = rowMap(keyFieldName).toString + RowWithData(keyValue, rowMap) + }) + + //update table with table row? + var table = new FakeInMemoryTable("SchemaMapTest", tableDef) + tableRows.foreach(row => table.processUpdate(row.key, row)) + + //assert on reading the table row - is that possible or need to use mock table with table interface + var existingRows = table.pullAllRows() + + //todo different for java + private def mapToEntity(rowData: List[Any]): SchemaTestData = + getListToObjectConverter[SchemaTestData](SchemaTestData)(rowData) + + +} +//copy of one in org.finos.vuu.example.ignite.utils +//todo if not going to use or move to common place +object getListToObjectConverter { + def apply[ReturnType](obj: Object): List[_] => ReturnType = { + val converter = obj.getClass.getMethods.find(x => x.getName == "apply" && x.isBridge).get + values => converter.invoke(obj, values.map(_.asInstanceOf[AnyRef]): _*).asInstanceOf[ReturnType] + } +} + + From 5d63f20afc7820e870e6b68077612dbb66492d17 Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Fri, 3 May 2024 18:03:01 +0100 Subject: [PATCH 03/19] #1318 notes on next things to try --- .../org/finos/vuu/util/SchemaExample.scala | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala b/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala index 37e9734ef..e3465c371 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala @@ -7,11 +7,15 @@ import org.finos.vuu.util.schema.{ExternalEntitySchema, ExternalEntitySchemaBuil class SchemaExample { - //todo - try in java - //try more scenarios - // - virtual table - filter, type ahead - // - error scenarios fetching data or converting types - //error handling - review api for returning error + //todo - do we want to turn this in to functional test so it fails if we break it while changing any part of the schema mapper + //todo try more scenarios + // - virtual table - include filter, type ahead + // - error scenarios fetching data or converting types - review api for returning error + // - when table def has fewer columns than fields on entity + // - when table def has different named column from field on entity + // - when table def has different typed column from field on entity + // - when table def columns are in different order from fields on entity + // - example of generating table Def from an class rather than using column builder? maybe more of a unit teest/example for table def? //create table def (use column builder) val tableDef = TableDef( @@ -23,7 +27,7 @@ class SchemaExample { .build ) - //todo to respect the QueryEntity order of fields, should be generated using that? + //todo to respect the QueryEntity order of fields, if it is different from order of fields on the entity class, should be generated using that? //create entity schema val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() .withEntity(classOf[SchemaTestData]) @@ -49,11 +53,13 @@ class SchemaExample { .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) // map to entity object - as order of values are relevant to how the query schema was defined + //todo two options, is direct to row map better if query result returns values? val tableRowMap1 = result .map(rowData => mapToEntity(rowData)) .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)) - private val tableRowMap2: Seq[Map[String, Any]] = result.map(rowData => schemaMapper.toInternalRowMap(rowData)) + private val tableRowMap2: Seq[Map[String, Any]] = + result.map(rowData => schemaMapper.toInternalRowMap(rowData)) //map to tablerow val keyFieldName = tableDef.keyField From d9c930a7fb94303dce9876ec11a5fdffeb4d8f00 Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Mon, 13 May 2024 13:53:18 +0100 Subject: [PATCH 04/19] #1318 java example of using schema mapper --- .../vuu/util/ScalaCollectionConverter.java | 1 + .../java/org/finos/vuu/util/ScalaList.java | 11 +++ .../org/finos/vuu/util/SchemaJavaExample.java | 90 +++++++++++++++++++ .../finos/vuu/util/SchemaJavaTestData.java | 7 ++ 4 files changed, 109 insertions(+) create mode 100644 vuu/src/main/java/org/finos/vuu/util/ScalaList.java create mode 100644 vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java create mode 100644 vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java diff --git a/vuu/src/main/java/org/finos/vuu/util/ScalaCollectionConverter.java b/vuu/src/main/java/org/finos/vuu/util/ScalaCollectionConverter.java index 1987683bf..194f9f8ea 100644 --- a/vuu/src/main/java/org/finos/vuu/util/ScalaCollectionConverter.java +++ b/vuu/src/main/java/org/finos/vuu/util/ScalaCollectionConverter.java @@ -27,3 +27,4 @@ public static List toJava(scala.collection.immutable.List l) { return CollectionConverters.SeqHasAsJava(l).asJava(); } } + diff --git a/vuu/src/main/java/org/finos/vuu/util/ScalaList.java b/vuu/src/main/java/org/finos/vuu/util/ScalaList.java new file mode 100644 index 000000000..db7b7de33 --- /dev/null +++ b/vuu/src/main/java/org/finos/vuu/util/ScalaList.java @@ -0,0 +1,11 @@ +package org.finos.vuu.util; + +import java.util.Arrays; + +import static org.finos.vuu.util.ScalaCollectionConverter.toScala; + +public class ScalaList { + public static scala.collection.Iterable of(T... args) { + return toScala(Arrays.asList(args)); + } +} diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java new file mode 100644 index 000000000..5495942c1 --- /dev/null +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java @@ -0,0 +1,90 @@ +package org.finos.vuu.util; + +import org.finos.vuu.api.ColumnBuilder; +import org.finos.vuu.api.TableDef; +import org.finos.vuu.test.FakeIgniteStore; +import org.finos.vuu.test.SchemaTestData; +import org.finos.vuu.util.schema.ExternalEntitySchemaBuilder; +import org.finos.vuu.util.schema.SchemaMapperBuilder; +import org.finos.vuu.util.schema.SchemaMapperBuilder$; +import scala.collection.immutable.Seq; + +import java.util.List; + +import static org.finos.vuu.util.ScalaCollectionConverter.toScala; +import static org.finos.vuu.util.ScalaCollectionConverter.toScalaSeq; + +public class SchemaJavaExample { + public void doSomething(){ + + //create table def + var tableDef = TableDef.apply( + "MyExampleTable", + "Id", + new ColumnBuilder() + .addString("Id") + .addDouble("NotionalValue") + .build(), + toScalaSeq(List.of()) + ); + + + //create entity schema + var externalEntitySchema = ExternalEntitySchemaBuilder.apply() + .withEntity(SchemaJavaTestData.class) + .withIndex("ID_INDEX", toScala( List.of("id"))) + .build(); + + //create schema mapper + var schemaMapper = SchemaMapperBuilder.apply(externalEntitySchema, tableDef.columns()) + //.withFieldsMap(columnNameByExternalField) + .build(); + + //get data from ignite as list of varues + var queryName = "myQuery"; + var igniteStore = new FakeIgniteStore(); + igniteStore.setUpSqlFieldsQuery( + queryName, + toScala(List.of( + toScala(List.of("id1", 10.5)) + )) +// ScalaList.of( +// ScalaList.of("id1", 10.5) +// ) + ); + +// List> result = igniteStore.getSqlFieldsQuery(queryName) +// .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")); + + // map to entity object - as order of values are relevant to how the query schema was defined +// var tableRowMap1 = result +// .map(rowData => mapToEntity(rowData)) +// .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)); + +// var tableRowMap2 = +// result.map(rowData => schemaMapper.toInternalRowMap(rowData)); +// +// //map to tablerow +// var keyFieldName = tableDef.keyField; +// var tableRows = tableRowMap2.map(rowMap => { +// var keyvarue = rowMap(keyFieldName).toString +// RowWithData(keyvarue, rowMap) +// }); +// +// //update table with table row? +// var table = new FakeInMemoryTable("SchemaMapTest", tableDef); +// tableRows.foreach(row => table.processUpdate(row.key, row)); +// +// //assert on reading the table row - is that possible or need to use mock table with table interface +// var existingRows = table.pullAllRows(); + } + + + //todo different for java +// private def mapToEntity(rowData: List[Any]): SchemaTestData = +// getListToObjectConverter[SchemaTestData](SchemaTestData)(rowData) + + +} + + diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java new file mode 100644 index 000000000..83139d427 --- /dev/null +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java @@ -0,0 +1,7 @@ +package org.finos.vuu.util; + +public class SchemaJavaTestData { + public String Id; + public double NotionalValue; + +} From a9df733f679f919755e42e92028c4bc132deecbe Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Mon, 13 May 2024 17:14:44 +0100 Subject: [PATCH 05/19] #1318 fleshing out java schema mapper example --- .../java/org/finos/vuu/util/ScalaList.java | 2 +- .../org/finos/vuu/util/SchemaJavaExample.java | 79 ++++++++++--------- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/vuu/src/main/java/org/finos/vuu/util/ScalaList.java b/vuu/src/main/java/org/finos/vuu/util/ScalaList.java index db7b7de33..11bf3b172 100644 --- a/vuu/src/main/java/org/finos/vuu/util/ScalaList.java +++ b/vuu/src/main/java/org/finos/vuu/util/ScalaList.java @@ -5,7 +5,7 @@ import static org.finos.vuu.util.ScalaCollectionConverter.toScala; public class ScalaList { - public static scala.collection.Iterable of(T... args) { + public static scala.collection.immutable.List of(T... args) { return toScala(Arrays.asList(args)); } } diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java index 5495942c1..80d90e716 100644 --- a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java @@ -2,20 +2,24 @@ import org.finos.vuu.api.ColumnBuilder; import org.finos.vuu.api.TableDef; +import org.finos.vuu.core.table.RowWithData; import org.finos.vuu.test.FakeIgniteStore; -import org.finos.vuu.test.SchemaTestData; +import org.finos.vuu.test.FakeInMemoryTable; import org.finos.vuu.util.schema.ExternalEntitySchemaBuilder; import org.finos.vuu.util.schema.SchemaMapperBuilder; -import org.finos.vuu.util.schema.SchemaMapperBuilder$; -import scala.collection.immutable.Seq; +import scala.jdk.javaapi.OptionConverters; import java.util.List; -import static org.finos.vuu.util.ScalaCollectionConverter.toScala; -import static org.finos.vuu.util.ScalaCollectionConverter.toScalaSeq; +import static org.finos.vuu.util.ScalaCollectionConverter.*; public class SchemaJavaExample { - public void doSomething(){ + + public static void main(String[] args) throws Exception { + + doSomething(); + } + public static void doSomething() throws Exception { //create table def var tableDef = TableDef.apply( @@ -32,7 +36,7 @@ public void doSomething(){ //create entity schema var externalEntitySchema = ExternalEntitySchemaBuilder.apply() .withEntity(SchemaJavaTestData.class) - .withIndex("ID_INDEX", toScala( List.of("id"))) + .withIndex("ID_INDEX", toScala( List.of("Id"))) .build(); //create schema mapper @@ -40,43 +44,42 @@ public void doSomething(){ //.withFieldsMap(columnNameByExternalField) .build(); - //get data from ignite as list of varues + //get data from ignite as list of values var queryName = "myQuery"; var igniteStore = new FakeIgniteStore(); igniteStore.setUpSqlFieldsQuery( queryName, - toScala(List.of( - toScala(List.of("id1", 10.5)) - )) -// ScalaList.of( -// ScalaList.of("id1", 10.5) -// ) + ScalaList.of(ScalaList.of("id1", 10.5)) ); -// List> result = igniteStore.getSqlFieldsQuery(queryName) -// .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")); - - // map to entity object - as order of values are relevant to how the query schema was defined -// var tableRowMap1 = result -// .map(rowData => mapToEntity(rowData)) -// .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)); - -// var tableRowMap2 = -// result.map(rowData => schemaMapper.toInternalRowMap(rowData)); -// -// //map to tablerow -// var keyFieldName = tableDef.keyField; -// var tableRows = tableRowMap2.map(rowMap => { -// var keyvarue = rowMap(keyFieldName).toString -// RowWithData(keyvarue, rowMap) -// }); -// -// //update table with table row? -// var table = new FakeInMemoryTable("SchemaMapTest", tableDef); -// tableRows.foreach(row => table.processUpdate(row.key, row)); -// -// //assert on reading the table row - is that possible or need to use mock table with table interface -// var existingRows = table.pullAllRows(); + //todo should use fake java store? + List> result = + OptionConverters.toJava(igniteStore.getSqlFieldsQuery(queryName)) + .map(listOfLists -> toJava(listOfLists.map(list -> toJava(list)).toList())) + .orElseThrow(()-> new Exception("query does not exist in store. make sure it is setup")); + + // map to entity object and then to row + + //map to row directly + var tableRowMap = result.stream() + .map(rowData -> + schemaMapper.toInternalRowMap(toScala(rowData) + )); + //.map(rowMap -> toJava(rowMap)); + + //map to tablerow + var keyFieldName = tableDef.keyField(); + var tableRows = tableRowMap.map(rowMap -> { + var keyValue = rowMap.get(keyFieldName).toString(); + return new RowWithData(keyValue, rowMap); + }); + + //update table with table row? + var table = new FakeInMemoryTable("SchemaMapJavaTest", tableDef); + tableRows.forEach(row -> table.processUpdate(row.key(), row, 0)); //todo use clock now + + //assert on reading the table row - is that possible or need to use mock table with table interface + var existingRows = table.pullAllRows(); } From d6aedccbaa59cfd19979d41abec781f062f36b66 Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Mon, 13 May 2024 17:14:58 +0100 Subject: [PATCH 06/19] #1318 fleshing out scala schema mapper example --- .../scala/org/finos/vuu/api/ColumnBuilder.scala | 10 +++++----- .../scala/org/finos/vuu/test/FakeIgniteStore.scala | 2 +- .../org/finos/vuu/test/FakeInMemoryTable.scala | 4 ++-- .../scala/org/finos/vuu/util/SchemaExample.scala | 8 ++++---- .../vuu/util/schema/SchemaFunctionalTest.scala | 13 +++++++++++++ 5 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala diff --git a/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala index 870800258..de742114a 100644 --- a/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala +++ b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala @@ -9,24 +9,24 @@ class ColumnBuilder { val columns = new ArrayBuilder.ofRef[String]() def addString(columnName: String): ColumnBuilder = { - columns +: (columnName + ":String") + columns += (columnName + ":String") this } def addDouble(columnName: String): ColumnBuilder = { - columns +: (columnName + ":Double") + columns += (columnName + ":Double") this } def addInt(columnName: String): ColumnBuilder = { - columns +: (columnName + ":Int") + columns += (columnName + ":Int") this } def addLong(columnName: String): ColumnBuilder = { - columns +: (columnName + ":Long") + columns += (columnName + ":Long") this } - def build: Array[Column] = Columns.fromNames(columns.result()) + def build(): Array[Column] = Columns.fromNames(columns.result()) } diff --git a/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala b/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala index 8b89de14b..74e303640 100644 --- a/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala +++ b/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala @@ -21,5 +21,5 @@ class FakeIgniteStore(){ } } -case class SchemaTestData(id :String, notionalValue: Double) { +case class SchemaTestData(Id :String, NotionalValue: Double) { } \ No newline at end of file diff --git a/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala b/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala index a4275bc02..0fc8fb3ae 100644 --- a/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala +++ b/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala @@ -5,11 +5,11 @@ import org.finos.vuu.core.index.IndexedField import org.finos.vuu.core.table.{Column, ColumnValueProvider, DataTable, KeyObserver, RowData, RowKeyUpdate, RowWithData, TableData, TablePrimaryKeys} import org.finos.vuu.viewport.{RowProcessor, ViewPortColumns} -class FakeInMemoryTable(name:String, tableDef: TableDef) extends DataTable { +class FakeInMemoryTable(val instanceName: String, val tableDef: TableDef) extends DataTable { private val rowMap = scala.collection.mutable.HashMap.empty[String, RowWithData] - override def name: String = name + override def name: String = instanceName override def getTableDef: TableDef = tableDef override def processUpdate(rowKey: String, rowUpdate: RowWithData, timeStamp: Long = 0): Unit = diff --git a/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala b/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala index e3465c371..c213d026c 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala @@ -24,14 +24,14 @@ class SchemaExample { columns = new ColumnBuilder() .addString("Id") .addDouble("NotionalValue") - .build + .build() ) //todo to respect the QueryEntity order of fields, if it is different from order of fields on the entity class, should be generated using that? //create entity schema val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() .withEntity(classOf[SchemaTestData]) - .withIndex("ID_INDEX", List("id")) + .withIndex("ID_INDEX", List("Id")) .build() //create schema mapper @@ -44,8 +44,8 @@ class SchemaExample { val igniteStore: FakeIgniteStore = new FakeIgniteStore igniteStore.setUpSqlFieldsQuery( queryName, - List[List[Any]]( - "id1", 10.5 + List( + List("id1", 10.5) ) ) diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala new file mode 100644 index 000000000..f19d149da --- /dev/null +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala @@ -0,0 +1,13 @@ +package org.finos.vuu.util.schema + +import org.finos.vuu.util.SchemaExample +import org.scalatest.featurespec.AnyFeatureSpec +class SchemaFunctionalTest extends AnyFeatureSpec { + + Feature("Schema Functional test") { + Scenario("Manually define table columns, Generate external schema from class object, use default index based mapping and update in memory table") { + + var example = new SchemaExample() + } + } +} From 2df15a4531f49db048c1fd5d5a85622119866b81 Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Mon, 13 May 2024 17:39:57 +0100 Subject: [PATCH 07/19] #1318 refactoring scala example into test --- .../org/finos/vuu/api/ColumnBuilder.scala | 4 +- .../org/finos/vuu/util/SchemaJavaExample.java | 2 +- .../finos/vuu/test/FakeInMemoryTable.scala | 3 +- .../org/finos/vuu/util/SchemaExample.scala | 102 ++++++++---------- .../util/schema/SchemaFunctionalTest.scala | 38 ++++++- 5 files changed, 84 insertions(+), 65 deletions(-) diff --git a/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala index de742114a..b1e06b4b1 100644 --- a/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala +++ b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala @@ -2,11 +2,11 @@ package org.finos.vuu.api import org.finos.vuu.core.table.{Column, Columns} -import scala.collection.mutable.ArrayBuilder +import scala.collection.mutable class ColumnBuilder { - val columns = new ArrayBuilder.ofRef[String]() + val columns = new mutable.ArrayBuilder.ofRef[String]() def addString(columnName: String): ColumnBuilder = { columns += (columnName + ":String") diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java index 80d90e716..ea1f16681 100644 --- a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java @@ -55,7 +55,7 @@ public static void doSomething() throws Exception { //todo should use fake java store? List> result = OptionConverters.toJava(igniteStore.getSqlFieldsQuery(queryName)) - .map(listOfLists -> toJava(listOfLists.map(list -> toJava(list)).toList())) + .map(listOfLists -> toJava(listOfLists.map(ScalaCollectionConverter::toJava).toList())) .orElseThrow(()-> new Exception("query does not exist in store. make sure it is setup")); // map to entity object and then to row diff --git a/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala b/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala index 0fc8fb3ae..c01ed8efc 100644 --- a/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala +++ b/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala @@ -16,8 +16,7 @@ class FakeInMemoryTable(val instanceName: String, val tableDef: TableDef) extend rowMap += (rowKey -> rowUpdate) override def pullRow(key: String): RowData = - rowMap.get(key) - .getOrElse(throw new Exception(s"Could not find row data for key $key in table $name")) + rowMap.getOrElse(key, throw new Exception(s"Could not find row data for key $key in table $name")) def pullAllRows() : List[RowWithData] = rowMap.values.toList diff --git a/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala b/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala index c213d026c..daf75a68d 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala @@ -1,11 +1,14 @@ package org.finos.vuu.util -import org.finos.vuu.api.{ColumnBuilder, TableDef} +import org.finos.vuu.api.TableDef import org.finos.vuu.core.table.RowWithData import org.finos.vuu.test.{FakeIgniteStore, FakeInMemoryTable, SchemaTestData} -import org.finos.vuu.util.schema.{ExternalEntitySchema, ExternalEntitySchemaBuilder, SchemaMapperBuilder} +import org.finos.vuu.util.schema.SchemaMapper -class SchemaExample { +class SchemaExample(val tableDef: TableDef, schemaMapper: SchemaMapper) { + + val igniteStore: FakeIgniteStore = new FakeIgniteStore + var table = new FakeInMemoryTable("SchemaMapTest", tableDef) //todo - do we want to turn this in to functional test so it fails if we break it while changing any part of the schema mapper //todo try more scenarios @@ -17,63 +20,43 @@ class SchemaExample { // - when table def columns are in different order from fields on entity // - example of generating table Def from an class rather than using column builder? maybe more of a unit teest/example for table def? - //create table def (use column builder) - val tableDef = TableDef( - name = "MyExampleTable", - keyField = "Id", - columns = new ColumnBuilder() - .addString("Id") - .addDouble("NotionalValue") - .build() - ) - - //todo to respect the QueryEntity order of fields, if it is different from order of fields on the entity class, should be generated using that? - //create entity schema - val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() - .withEntity(classOf[SchemaTestData]) - .withIndex("ID_INDEX", List("Id")) - .build() - - //create schema mapper - private val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) - //.withFieldsMap(columnNameByExternalField) - .build() - - //get data from ignite as list of values - private val queryName = "myQuery" - val igniteStore: FakeIgniteStore = new FakeIgniteStore - igniteStore.setUpSqlFieldsQuery( - queryName, - List( - List("id1", 10.5) - ) - ) - - val result = igniteStore.getSqlFieldsQuery(queryName) - .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) - - // map to entity object - as order of values are relevant to how the query schema was defined - //todo two options, is direct to row map better if query result returns values? - val tableRowMap1 = result - .map(rowData => mapToEntity(rowData)) - .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)) - - private val tableRowMap2: Seq[Map[String, Any]] = - result.map(rowData => schemaMapper.toInternalRowMap(rowData)) - - //map to tablerow - val keyFieldName = tableDef.keyField - val tableRows = tableRowMap2.map(rowMap => { - val keyValue = rowMap(keyFieldName).toString - RowWithData(keyValue, rowMap) - }) - - //update table with table row? - var table = new FakeInMemoryTable("SchemaMapTest", tableDef) - tableRows.foreach(row => table.processUpdate(row.key, row)) - //assert on reading the table row - is that possible or need to use mock table with table interface - var existingRows = table.pullAllRows() + //todo make it not specific to ignite? type the results more? + def givenIgniteSqlFieldQueryReturns(queryName: String, results: List[List[Any]]): Unit = { + igniteStore.setUpSqlFieldsQuery(queryName, results) + } + + def getIgniteQueryResult(queryName: String): List[List[Any]] = { + igniteStore.getSqlFieldsQuery(queryName) + .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) + } + + def mapToRow(resultData: List[List[Any]]): Seq[RowWithData] = { + + // map to entity object - as order of values are relevant to how the query schema was defined + //todo two options, is direct to row map better if query result returns values? + val tableRowMap1 = resultData + .map(rowData => mapToEntity(rowData)) + .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)) + + val tableRowMap2: Seq[Map[String, Any]] = + resultData.map(rowData => schemaMapper.toInternalRowMap(rowData)) + + //map to tablerow + val keyFieldName = tableDef.keyField + tableRowMap2.map(rowMap => { + val keyValue = rowMap(keyFieldName).toString + RowWithData(keyValue, rowMap) + }) + } + + def updateTable(rows: Seq[RowWithData]): Unit = { + rows.foreach(row => table.processUpdate(row.key, row)) + } + + def getExitingRows() = { + table.pullAllRows() + } //todo different for java private def mapToEntity(rowData: List[Any]): SchemaTestData = @@ -81,6 +64,7 @@ class SchemaExample { } + //copy of one in org.finos.vuu.example.ignite.utils //todo if not going to use or move to common place object getListToObjectConverter { diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala index f19d149da..d52acbd66 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala @@ -1,5 +1,7 @@ package org.finos.vuu.util.schema +import org.finos.vuu.api.{ColumnBuilder, TableDef} +import org.finos.vuu.test.SchemaTestData import org.finos.vuu.util.SchemaExample import org.scalatest.featurespec.AnyFeatureSpec class SchemaFunctionalTest extends AnyFeatureSpec { @@ -7,7 +9,41 @@ class SchemaFunctionalTest extends AnyFeatureSpec { Feature("Schema Functional test") { Scenario("Manually define table columns, Generate external schema from class object, use default index based mapping and update in memory table") { - var example = new SchemaExample() + val tableDef = TableDef( + name = "MyExampleTable", + keyField = "Id", + columns = new ColumnBuilder() + .addString("Id") + .addDouble("NotionalValue") + .build() + ) + + //todo to respect the QueryEntity order of fields, if it is different from order of fields on the entity class, should be generated using that? + val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() + .withEntity(classOf[SchemaTestData]) + .withIndex("ID_INDEX", List("Id")) + .build() + + //create schema mapper + val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + //.withFieldsMap(columnNameByExternalField) + .build() + + + val example = new SchemaExample(tableDef, schemaMapper) + + val queryName = "myQuery" + example.givenIgniteSqlFieldQueryReturns(queryName, List(List("id1", 10.5))) + + val result = example.getIgniteQueryResult(queryName) + val rows = example.mapToRow(result) + rows.foreach(row => example.table.processUpdate(row.key, row)) + + val existingTableRows = example.table.pullAllRows() + + assert(existingTableRows.size == 1) + assert(existingTableRows.head.get("NotionalValue") == 10.5) + } } } From e94cf769e79a9bd1e46b398dec2dff5784fc7ba7 Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Tue, 14 May 2024 17:29:28 +0100 Subject: [PATCH 08/19] #1318 added more scenario for schema mapper testing --- .../org/finos/vuu/util/SchemaJavaExample.java | 13 +- .../finos/vuu/util/SchemaJavaTestData.java | 1 - .../org/finos/vuu/test/FakeIgniteStore.scala | 11 +- .../org/finos/vuu/util/SchemaExample.scala | 27 ++-- .../util/schema/SchemaFunctionalTest.scala | 143 ++++++++++++++++-- 5 files changed, 154 insertions(+), 41 deletions(-) diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java index ea1f16681..e67de118b 100644 --- a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java @@ -46,7 +46,7 @@ public static void doSomething() throws Exception { //get data from ignite as list of values var queryName = "myQuery"; - var igniteStore = new FakeIgniteStore(); + var igniteStore = new FakeIgniteStore(); igniteStore.setUpSqlFieldsQuery( queryName, ScalaList.of(ScalaList.of("id1", 10.5)) @@ -70,8 +70,9 @@ public static void doSomething() throws Exception { //map to tablerow var keyFieldName = tableDef.keyField(); var tableRows = tableRowMap.map(rowMap -> { - var keyValue = rowMap.get(keyFieldName).toString(); - return new RowWithData(keyValue, rowMap); + var keyOptional = OptionConverters.toJava(rowMap.get(keyFieldName)); + var key = keyOptional.orElseThrow(); + return new RowWithData(key.toString(), rowMap); }); //update table with table row? @@ -82,12 +83,6 @@ public static void doSomething() throws Exception { var existingRows = table.pullAllRows(); } - - //todo different for java -// private def mapToEntity(rowData: List[Any]): SchemaTestData = -// getListToObjectConverter[SchemaTestData](SchemaTestData)(rowData) - - } diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java index 83139d427..250c3c475 100644 --- a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java @@ -3,5 +3,4 @@ public class SchemaJavaTestData { public String Id; public double NotionalValue; - } diff --git a/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala b/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala index 74e303640..36045e915 100644 --- a/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala +++ b/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala @@ -1,14 +1,14 @@ package org.finos.vuu.test -class FakeIgniteStore(){ +class FakeIgniteStore[T](){ - private val queryToEntityResultMap = scala.collection.mutable.HashMap.empty[String, List[SchemaTestData]] + private val queryToEntityResultMap = scala.collection.mutable.HashMap.empty[String, List[T]] private val queryToValuesResultMap = scala.collection.mutable.HashMap.empty[String, List[List[Any]]] - def setUpIndexQuery(queryName:String, queryResult:List[SchemaTestData]): Unit = { + def setUpIndexQuery(queryName:String, queryResult:List[T]): Unit = { queryToEntityResultMap += (queryName -> queryResult) } - def getIndexQuery(queryName: String): Option[List[SchemaTestData]] = { + def getIndexQuery(queryName: String): Option[List[T]] = { queryToEntityResultMap.get(queryName) } @@ -19,7 +19,4 @@ class FakeIgniteStore(){ def getSqlFieldsQuery(queryName: String): Option[List[List[Any]]] = { queryToValuesResultMap.get(queryName) } -} - -case class SchemaTestData(Id :String, NotionalValue: Double) { } \ No newline at end of file diff --git a/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala b/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala index daf75a68d..d8a6a09e1 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala @@ -2,24 +2,22 @@ package org.finos.vuu.util import org.finos.vuu.api.TableDef import org.finos.vuu.core.table.RowWithData -import org.finos.vuu.test.{FakeIgniteStore, FakeInMemoryTable, SchemaTestData} +import org.finos.vuu.test.{FakeIgniteStore, FakeInMemoryTable} import org.finos.vuu.util.schema.SchemaMapper class SchemaExample(val tableDef: TableDef, schemaMapper: SchemaMapper) { - val igniteStore: FakeIgniteStore = new FakeIgniteStore + val igniteStore: FakeIgniteStore[SchemaTestData] = new FakeIgniteStore[SchemaTestData] var table = new FakeInMemoryTable("SchemaMapTest", tableDef) - //todo - do we want to turn this in to functional test so it fails if we break it while changing any part of the schema mapper //todo try more scenarios // - virtual table - include filter, type ahead // - error scenarios fetching data or converting types - review api for returning error - // - when table def has fewer columns than fields on entity - // - when table def has different named column from field on entity - // - when table def has different typed column from field on entity - // - when table def columns are in different order from fields on entity // - example of generating table Def from an class rather than using column builder? maybe more of a unit teest/example for table def? + //todo when query result has less number of fields than table column + //todo when fields and column diff order but no mapping specified - assert on error message being useful + //todo make it not specific to ignite? type the results more? def givenIgniteSqlFieldQueryReturns(queryName: String, results: List[List[Any]]): Unit = { @@ -35,9 +33,9 @@ class SchemaExample(val tableDef: TableDef, schemaMapper: SchemaMapper) { // map to entity object - as order of values are relevant to how the query schema was defined //todo two options, is direct to row map better if query result returns values? - val tableRowMap1 = resultData - .map(rowData => mapToEntity(rowData)) - .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)) +// val tableRowMap1 = resultData +// .map(rowData => mapToEntity(rowData)) +// .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)) val tableRowMap2: Seq[Map[String, Any]] = resultData.map(rowData => schemaMapper.toInternalRowMap(rowData)) @@ -58,10 +56,9 @@ class SchemaExample(val tableDef: TableDef, schemaMapper: SchemaMapper) { table.pullAllRows() } - //todo different for java - private def mapToEntity(rowData: List[Any]): SchemaTestData = - getListToObjectConverter[SchemaTestData](SchemaTestData)(rowData) - +// //todo different for java +// private def mapToEntity(rowData: List[Any]): SchemaTestData = +// getListToObjectConverter[SchemaTestData](SchemaTestData)(rowData) } @@ -74,4 +71,6 @@ object getListToObjectConverter { } } +case class SchemaTestData(id :String, clientId:Int, notionalValue: Double) {} + diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala index d52acbd66..581a94d13 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala @@ -1,27 +1,27 @@ package org.finos.vuu.util.schema import org.finos.vuu.api.{ColumnBuilder, TableDef} -import org.finos.vuu.test.SchemaTestData -import org.finos.vuu.util.SchemaExample +import org.finos.vuu.util.{SchemaExample, SchemaTestData} import org.scalatest.featurespec.AnyFeatureSpec class SchemaFunctionalTest extends AnyFeatureSpec { - Feature("Schema Functional test") { - Scenario("Manually define table columns, Generate external schema from class object, use default index based mapping and update in memory table") { + Feature("Update in memory table using schema mapper") { + Scenario("When table columns and entity fields match exactly") { val tableDef = TableDef( name = "MyExampleTable", - keyField = "Id", + keyField = "id", columns = new ColumnBuilder() - .addString("Id") - .addDouble("NotionalValue") + .addString("id") + .addInt("clientId") + .addDouble("notionalValue") .build() ) //todo to respect the QueryEntity order of fields, if it is different from order of fields on the entity class, should be generated using that? val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() .withEntity(classOf[SchemaTestData]) - .withIndex("ID_INDEX", List("Id")) + .withIndex("ID_INDEX", List("id")) .build() //create schema mapper @@ -29,11 +29,89 @@ class SchemaFunctionalTest extends AnyFeatureSpec { //.withFieldsMap(columnNameByExternalField) .build() + val example = new SchemaExample(tableDef, schemaMapper) + + val queryName = "myQuery" + example.givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) + + val result = example.getIgniteQueryResult(queryName) + val rows = example.mapToRow(result) + rows.foreach(row => example.table.processUpdate(row.key, row)) + + val existingTableRows = example.table.pullAllRows() + + assert(existingTableRows.size == 1) + assert(existingTableRows.head.get("id") == "testId1") + assert(existingTableRows.head.get("clientId") == 5) + assert(existingTableRows.head.get("notionalValue") == 10.5) + } + + Scenario("When table has fewer columns than fields on external entity") { + + val tableDef = TableDef( + name = "MyExampleTable", + keyField = "id", + columns = new ColumnBuilder() + .addString("id") + .addDouble("notionalValue") + .build() + ) + + val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() + .withEntity(classOf[SchemaTestData]) + .withIndex("ID_INDEX", List("id")) + .build() + + val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + .withFieldsMap( + Map( + "id" -> "id", + "notionalValue" -> "notionalValue", + ) + ) + .build() + + val example = new SchemaExample(tableDef, schemaMapper) + + val queryName = "myQuery" + example.givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) + + val result = example.getIgniteQueryResult(queryName) + val rows = example.mapToRow(result) + rows.foreach(row => example.table.processUpdate(row.key, row)) + + val existingTableRows = example.table.pullAllRows() + + + assert(existingTableRows.size == 1) + assert(existingTableRows.head.get("id") == "testId1") + assert(existingTableRows.head.get("notionalValue") == 10.5) + } + + Scenario("When table has columns with different name than fields on external entity") { + + val tableDef = TableDef( + name = "MyExampleTable", + keyField = "firstColumn", + columns = new ColumnBuilder() + .addString("firstColumn") + .addInt("secondColumn") + .addDouble("thirdColumn") + .build() + ) + + val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() + .withEntity(classOf[SchemaTestData]) + .withIndex("ID_INDEX", List("id")) + .build() + + val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + .build() val example = new SchemaExample(tableDef, schemaMapper) val queryName = "myQuery" - example.givenIgniteSqlFieldQueryReturns(queryName, List(List("id1", 10.5))) + example.givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) val result = example.getIgniteQueryResult(queryName) val rows = example.mapToRow(result) @@ -42,8 +120,53 @@ class SchemaFunctionalTest extends AnyFeatureSpec { val existingTableRows = example.table.pullAllRows() assert(existingTableRows.size == 1) - assert(existingTableRows.head.get("NotionalValue") == 10.5) + assert(existingTableRows.head.get("firstColumn") == "testId1") + assert(existingTableRows.head.get("secondColumn") == 5) + assert(existingTableRows.head.get("thirdColumn") == 10.5) + } + + Scenario("When table has columns are in different order from fields on external entity") { + + val tableDef = TableDef( + name = "MyExampleTable", + keyField = "id", + columns = new ColumnBuilder() + .addDouble("notionalValue") + .addString("id") + .addInt("clientId") + .build() + ) + + val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() + .withEntity(classOf[SchemaTestData]) + .withIndex("ID_INDEX", List("id")) + .build() + + val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + .withFieldsMap( + Map( + "notionalValue" -> "notionalValue", + "id" -> "id", + "clientId" -> "clientId", + ) + ) + .build() + + val example = new SchemaExample(tableDef, schemaMapper) + + val queryName = "myQuery" + example.givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) + + val result = example.getIgniteQueryResult(queryName) + val rows = example.mapToRow(result) + rows.foreach(row => example.table.processUpdate(row.key, row)) + val existingTableRows = example.table.pullAllRows() + + assert(existingTableRows.size == 1) + assert(existingTableRows.head.get("id") == "testId1") + assert(existingTableRows.head.get("clientId") == 5) + assert(existingTableRows.head.get("notionalValue") == 10.5) } } } From 0c5f04c3ee1463d1e1e125e0a148f79391bbbe6d Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Thu, 16 May 2024 15:02:34 +0100 Subject: [PATCH 09/19] #1318 refactor schema mapper functional test to introduce base class --- .../org/finos/vuu/util/SchemaJavaExample.java | 2 + .../finos/vuu/test/FakeInMemoryTable.scala | 2 +- .../org/finos/vuu/util/SchemaExample.scala | 76 ------------------ ...scala => SchemaMapperFunctionalTest.scala} | 63 +++++---------- .../SchemaMapperFunctionalTestBase.scala | 79 +++++++++++++++++++ 5 files changed, 103 insertions(+), 119 deletions(-) delete mode 100644 vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala rename vuu/src/test/scala/org/finos/vuu/util/schema/{SchemaFunctionalTest.scala => SchemaMapperFunctionalTest.scala} (63%) create mode 100644 vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java index e67de118b..d9d7817cc 100644 --- a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java @@ -44,6 +44,8 @@ public static void doSomething() throws Exception { //.withFieldsMap(columnNameByExternalField) .build(); + + //get data from ignite as list of values var queryName = "myQuery"; var igniteStore = new FakeIgniteStore(); diff --git a/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala b/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala index c01ed8efc..7898fdb27 100644 --- a/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala +++ b/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala @@ -12,7 +12,7 @@ class FakeInMemoryTable(val instanceName: String, val tableDef: TableDef) extend override def name: String = instanceName override def getTableDef: TableDef = tableDef - override def processUpdate(rowKey: String, rowUpdate: RowWithData, timeStamp: Long = 0): Unit = + override def processUpdate(rowKey: String, rowUpdate: RowWithData, timeStamp: Long): Unit = rowMap += (rowKey -> rowUpdate) override def pullRow(key: String): RowData = diff --git a/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala b/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala deleted file mode 100644 index d8a6a09e1..000000000 --- a/vuu/src/test/scala/org/finos/vuu/util/SchemaExample.scala +++ /dev/null @@ -1,76 +0,0 @@ -package org.finos.vuu.util - -import org.finos.vuu.api.TableDef -import org.finos.vuu.core.table.RowWithData -import org.finos.vuu.test.{FakeIgniteStore, FakeInMemoryTable} -import org.finos.vuu.util.schema.SchemaMapper - -class SchemaExample(val tableDef: TableDef, schemaMapper: SchemaMapper) { - - val igniteStore: FakeIgniteStore[SchemaTestData] = new FakeIgniteStore[SchemaTestData] - var table = new FakeInMemoryTable("SchemaMapTest", tableDef) - - //todo try more scenarios - // - virtual table - include filter, type ahead - // - error scenarios fetching data or converting types - review api for returning error - // - example of generating table Def from an class rather than using column builder? maybe more of a unit teest/example for table def? - - //todo when query result has less number of fields than table column - //todo when fields and column diff order but no mapping specified - assert on error message being useful - - - //todo make it not specific to ignite? type the results more? - def givenIgniteSqlFieldQueryReturns(queryName: String, results: List[List[Any]]): Unit = { - igniteStore.setUpSqlFieldsQuery(queryName, results) - } - - def getIgniteQueryResult(queryName: String): List[List[Any]] = { - igniteStore.getSqlFieldsQuery(queryName) - .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) - } - - def mapToRow(resultData: List[List[Any]]): Seq[RowWithData] = { - - // map to entity object - as order of values are relevant to how the query schema was defined - //todo two options, is direct to row map better if query result returns values? -// val tableRowMap1 = resultData -// .map(rowData => mapToEntity(rowData)) -// .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)) - - val tableRowMap2: Seq[Map[String, Any]] = - resultData.map(rowData => schemaMapper.toInternalRowMap(rowData)) - - //map to tablerow - val keyFieldName = tableDef.keyField - tableRowMap2.map(rowMap => { - val keyValue = rowMap(keyFieldName).toString - RowWithData(keyValue, rowMap) - }) - } - - def updateTable(rows: Seq[RowWithData]): Unit = { - rows.foreach(row => table.processUpdate(row.key, row)) - } - - def getExitingRows() = { - table.pullAllRows() - } - -// //todo different for java -// private def mapToEntity(rowData: List[Any]): SchemaTestData = -// getListToObjectConverter[SchemaTestData](SchemaTestData)(rowData) - -} - -//copy of one in org.finos.vuu.example.ignite.utils -//todo if not going to use or move to common place -object getListToObjectConverter { - def apply[ReturnType](obj: Object): List[_] => ReturnType = { - val converter = obj.getClass.getMethods.find(x => x.getName == "apply" && x.isBridge).get - values => converter.invoke(obj, values.map(_.asInstanceOf[AnyRef]): _*).asInstanceOf[ReturnType] - } -} - -case class SchemaTestData(id :String, clientId:Int, notionalValue: Double) {} - - diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala similarity index 63% rename from vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala rename to vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala index 581a94d13..8fbbf4d61 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaFunctionalTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala @@ -1,9 +1,9 @@ package org.finos.vuu.util.schema import org.finos.vuu.api.{ColumnBuilder, TableDef} -import org.finos.vuu.util.{SchemaExample, SchemaTestData} -import org.scalatest.featurespec.AnyFeatureSpec -class SchemaFunctionalTest extends AnyFeatureSpec { +import org.finos.vuu.test.FakeInMemoryTable + +class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { Feature("Update in memory table using schema mapper") { Scenario("When table columns and entity fields match exactly") { @@ -18,28 +18,21 @@ class SchemaFunctionalTest extends AnyFeatureSpec { .build() ) - //todo to respect the QueryEntity order of fields, if it is different from order of fields on the entity class, should be generated using that? - val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() + val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() .withEntity(classOf[SchemaTestData]) .withIndex("ID_INDEX", List("id")) .build() - //create schema mapper val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) - //.withFieldsMap(columnNameByExternalField) .build() - val example = new SchemaExample(tableDef, schemaMapper) - - val queryName = "myQuery" - example.givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) + val table = new FakeInMemoryTable("SchemaMapTest", tableDef) - val result = example.getIgniteQueryResult(queryName) - val rows = example.mapToRow(result) - rows.foreach(row => example.table.processUpdate(row.key, row)) + givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) - val existingTableRows = example.table.pullAllRows() + getDataAndUpdateTable(table, schemaMapper, queryName) + val existingTableRows = table.pullAllRows() assert(existingTableRows.size == 1) assert(existingTableRows.head.get("id") == "testId1") assert(existingTableRows.head.get("clientId") == 5) @@ -71,18 +64,13 @@ class SchemaFunctionalTest extends AnyFeatureSpec { ) .build() - val example = new SchemaExample(tableDef, schemaMapper) - - val queryName = "myQuery" - example.givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) - - val result = example.getIgniteQueryResult(queryName) - val rows = example.mapToRow(result) - rows.foreach(row => example.table.processUpdate(row.key, row)) + val table = new FakeInMemoryTable("SchemaMapTest", tableDef) - val existingTableRows = example.table.pullAllRows() + givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) + getDataAndUpdateTable(table, schemaMapper, queryName) + val existingTableRows = table.pullAllRows() assert(existingTableRows.size == 1) assert(existingTableRows.head.get("id") == "testId1") assert(existingTableRows.head.get("notionalValue") == 10.5) @@ -108,17 +96,12 @@ class SchemaFunctionalTest extends AnyFeatureSpec { val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) .build() - val example = new SchemaExample(tableDef, schemaMapper) + val table = new FakeInMemoryTable("SchemaMapTest", tableDef) + givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) - val queryName = "myQuery" - example.givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) - - val result = example.getIgniteQueryResult(queryName) - val rows = example.mapToRow(result) - rows.foreach(row => example.table.processUpdate(row.key, row)) - - val existingTableRows = example.table.pullAllRows() + getDataAndUpdateTable(table, schemaMapper, queryName) + val existingTableRows = table.pullAllRows() assert(existingTableRows.size == 1) assert(existingTableRows.head.get("firstColumn") == "testId1") assert(existingTableRows.head.get("secondColumn") == 5) @@ -152,21 +135,17 @@ class SchemaFunctionalTest extends AnyFeatureSpec { ) .build() - val example = new SchemaExample(tableDef, schemaMapper) - - val queryName = "myQuery" - example.givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) + val table = new FakeInMemoryTable("SchemaMapTest", tableDef) + givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) - val result = example.getIgniteQueryResult(queryName) - val rows = example.mapToRow(result) - rows.foreach(row => example.table.processUpdate(row.key, row)) - - val existingTableRows = example.table.pullAllRows() + getDataAndUpdateTable(table, schemaMapper, queryName) + val existingTableRows = table.pullAllRows() assert(existingTableRows.size == 1) assert(existingTableRows.head.get("id") == "testId1") assert(existingTableRows.head.get("clientId") == 5) assert(existingTableRows.head.get("notionalValue") == 10.5) } } + } diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala new file mode 100644 index 000000000..b50f2f735 --- /dev/null +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala @@ -0,0 +1,79 @@ +package org.finos.vuu.util.schema + +import org.finos.vuu.api.TableDef +import org.finos.vuu.core.table.{DataTable, RowWithData} +import org.finos.vuu.test.FakeIgniteStore +import org.scalatest.BeforeAndAfterEach +import org.scalatest.featurespec.AnyFeatureSpec + +class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterEach { + + val igniteStore: FakeIgniteStore[SchemaTestData] = new FakeIgniteStore[SchemaTestData] + + protected var queryName: String = "myQuery" + + override def beforeEach(): Unit = { + queryName += java.util.UUID.randomUUID.toString // unique query name for each test run + } + + protected def getDataAndUpdateTable(table: DataTable, schemaMapper: SchemaMapper, queryName: String): Unit = { + val result = getIgniteQueryResult(queryName) + val rows = mapToRow(result, schemaMapper, table.getTableDef) + rows.foreach(row => table.processUpdate(row.key, row, 0)) + } + + //todo make it not specific to ignite? type the results more? + protected def givenIgniteSqlFieldQueryReturns(queryName: String, results: List[List[Any]]): Unit = { + igniteStore.setUpSqlFieldsQuery(queryName, results) + } + + protected def getIgniteQueryResult(queryName: String): List[List[Any]] = { + igniteStore.getSqlFieldsQuery(queryName) + .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) + } + + + def mapToRow(resultData: List[List[Any]], schemaMapper: SchemaMapper, tableDef: TableDef): Seq[RowWithData] = { + + // map to entity object - as order of values are relevant to how the query schema was defined + //todo two options, is direct to row map better if query result returns values? + // val tableRowMap1 = resultData + // .map(rowData => mapToEntity(rowData)) + // .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)) + + val tableRowMap2: Seq[Map[String, Any]] = + resultData.map(rowData => schemaMapper.toInternalRowMap(rowData)) + + //map to tablerow + val keyFieldName = tableDef.keyField + tableRowMap2.map(rowMap => { + val keyValue = rowMap(keyFieldName).toString + RowWithData(keyValue, rowMap) + }) + } + //todo try more scenarios + // - virtual table - include filter, type ahead + // - error scenarios fetching data or converting types - review api for returning error + // - example of generating table Def from an class rather than using column builder? maybe more of a unit teest/example for table def? + + //todo to respect the QueryEntity order of fields, if it is different from order of fields on the entity class, should be generated using that? + //todo when query result has less number of fields than table column + //todo when fields and column diff order but no mapping specified - assert on error message being useful + + + // //todo different for java + // private def mapToEntity(rowData: List[Any]): SchemaTestData = + // getListToObjectConverter[SchemaTestData](SchemaTestData)(rowData) + +} + +//copy of one in org.finos.vuu.example.ignite.utils +////todo if not going to use or move to common place +//object getListToObjectConverter { +// def apply[ReturnType](obj: Object): List[_] => ReturnType = { +// val converter = obj.getClass.getMethods.find(x => x.getName == "apply" && x.isBridge).get +// values => converter.invoke(obj, values.map(_.asInstanceOf[AnyRef]): _*).asInstanceOf[ReturnType] +// } +//} + +case class SchemaTestData(id :String, clientId:Int, notionalValue: Double) {} \ No newline at end of file From 2d1f5955cecf2aeec3024bc15b5973887099a2a3 Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Thu, 16 May 2024 15:36:46 +0100 Subject: [PATCH 10/19] #1318 using data source for the functional test more generic so it is agonostic to whether data is from ignite or rest api etc --- .../org/finos/vuu/api/ColumnBuilder.scala | 5 ++- .../org/finos/vuu/util/SchemaJavaExample.java | 8 ++-- ...IgniteStore.scala => FakeDataSource.scala} | 10 ++--- .../schema/SchemaMapperFunctionalTest.scala | 45 ++++--------------- .../SchemaMapperFunctionalTestBase.scala | 30 ++++++++----- 5 files changed, 40 insertions(+), 58 deletions(-) rename vuu/src/test/scala/org/finos/vuu/test/{FakeIgniteStore.scala => FakeDataSource.scala} (58%) diff --git a/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala index b1e06b4b1..534835d31 100644 --- a/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala +++ b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala @@ -26,7 +26,10 @@ class ColumnBuilder { def addLong(columnName: String): ColumnBuilder = { columns += (columnName + ":Long") this - } + + } def build(): Array[Column] = Columns.fromNames(columns.result()) } + + diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java index d9d7817cc..11bf88721 100644 --- a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java @@ -3,7 +3,7 @@ import org.finos.vuu.api.ColumnBuilder; import org.finos.vuu.api.TableDef; import org.finos.vuu.core.table.RowWithData; -import org.finos.vuu.test.FakeIgniteStore; +import org.finos.vuu.test.FakeDataSource; import org.finos.vuu.test.FakeInMemoryTable; import org.finos.vuu.util.schema.ExternalEntitySchemaBuilder; import org.finos.vuu.util.schema.SchemaMapperBuilder; @@ -48,15 +48,15 @@ public static void doSomething() throws Exception { //get data from ignite as list of values var queryName = "myQuery"; - var igniteStore = new FakeIgniteStore(); - igniteStore.setUpSqlFieldsQuery( + var igniteStore = new FakeDataSource(); + igniteStore.setUpResultAsListOfValues( queryName, ScalaList.of(ScalaList.of("id1", 10.5)) ); //todo should use fake java store? List> result = - OptionConverters.toJava(igniteStore.getSqlFieldsQuery(queryName)) + OptionConverters.toJava(igniteStore.getAsListOfValues(queryName)) .map(listOfLists -> toJava(listOfLists.map(ScalaCollectionConverter::toJava).toList())) .orElseThrow(()-> new Exception("query does not exist in store. make sure it is setup")); diff --git a/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala b/vuu/src/test/scala/org/finos/vuu/test/FakeDataSource.scala similarity index 58% rename from vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala rename to vuu/src/test/scala/org/finos/vuu/test/FakeDataSource.scala index 36045e915..f448f5e50 100644 --- a/vuu/src/test/scala/org/finos/vuu/test/FakeIgniteStore.scala +++ b/vuu/src/test/scala/org/finos/vuu/test/FakeDataSource.scala @@ -1,22 +1,22 @@ package org.finos.vuu.test -class FakeIgniteStore[T](){ +class FakeDataSource[T]{ private val queryToEntityResultMap = scala.collection.mutable.HashMap.empty[String, List[T]] private val queryToValuesResultMap = scala.collection.mutable.HashMap.empty[String, List[List[Any]]] - def setUpIndexQuery(queryName:String, queryResult:List[T]): Unit = { + def setUpResultAsObjects(queryName:String, queryResult:List[T]): Unit = { queryToEntityResultMap += (queryName -> queryResult) } - def getIndexQuery(queryName: String): Option[List[T]] = { + def getAsObjects(queryName: String): Option[List[T]] = { queryToEntityResultMap.get(queryName) } - def setUpSqlFieldsQuery(queryName: String, resultValues: List[List[Any]]): Unit = { + def setUpResultAsListOfValues(queryName: String, resultValues: List[List[Any]]): Unit = { queryToValuesResultMap += (queryName -> resultValues) } - def getSqlFieldsQuery(queryName: String): Option[List[List[Any]]] = { + def getAsListOfValues(queryName: String): Option[List[List[Any]]] = { queryToValuesResultMap.get(queryName) } } \ No newline at end of file diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala index 8fbbf4d61..7cd40681f 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala @@ -8,7 +8,8 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { Feature("Update in memory table using schema mapper") { Scenario("When table columns and entity fields match exactly") { - val tableDef = TableDef( + val externalEntitySchema: ExternalEntitySchema = createExternalEntitySchema + val tableDef = TableDef( name = "MyExampleTable", keyField = "id", columns = new ColumnBuilder() @@ -17,18 +18,10 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { .addDouble("notionalValue") .build() ) - - val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() - .withEntity(classOf[SchemaTestData]) - .withIndex("ID_INDEX", List("id")) - .build() - val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) .build() - val table = new FakeInMemoryTable("SchemaMapTest", tableDef) - - givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) + givenQueryReturns(queryName, List(List("testId1", 5, 10.5))) getDataAndUpdateTable(table, schemaMapper, queryName) @@ -49,12 +42,7 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { .addDouble("notionalValue") .build() ) - - val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() - .withEntity(classOf[SchemaTestData]) - .withIndex("ID_INDEX", List("id")) - .build() - + val externalEntitySchema: ExternalEntitySchema = createExternalEntitySchema val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) .withFieldsMap( Map( @@ -63,10 +51,8 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { ) ) .build() - val table = new FakeInMemoryTable("SchemaMapTest", tableDef) - - givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) + givenQueryReturns(queryName, List(List("testId1", 5, 10.5))) getDataAndUpdateTable(table, schemaMapper, queryName) @@ -87,17 +73,11 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { .addDouble("thirdColumn") .build() ) - - val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() - .withEntity(classOf[SchemaTestData]) - .withIndex("ID_INDEX", List("id")) - .build() - + val externalEntitySchema: ExternalEntitySchema = createExternalEntitySchema val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) .build() - val table = new FakeInMemoryTable("SchemaMapTest", tableDef) - givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) + givenQueryReturns(queryName, List(List("testId1", 5, 10.5))) getDataAndUpdateTable(table, schemaMapper, queryName) @@ -119,12 +99,7 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { .addInt("clientId") .build() ) - - val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() - .withEntity(classOf[SchemaTestData]) - .withIndex("ID_INDEX", List("id")) - .build() - + val externalEntitySchema: ExternalEntitySchema = createExternalEntitySchema val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) .withFieldsMap( Map( @@ -134,9 +109,8 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { ) ) .build() - val table = new FakeInMemoryTable("SchemaMapTest", tableDef) - givenIgniteSqlFieldQueryReturns(queryName, List(List("testId1", 5, 10.5))) + givenQueryReturns(queryName, List(List("testId1", 5, 10.5))) getDataAndUpdateTable(table, schemaMapper, queryName) @@ -147,5 +121,4 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { assert(existingTableRows.head.get("notionalValue") == 10.5) } } - } diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala index b50f2f735..262fe1c63 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala @@ -2,14 +2,13 @@ package org.finos.vuu.util.schema import org.finos.vuu.api.TableDef import org.finos.vuu.core.table.{DataTable, RowWithData} -import org.finos.vuu.test.FakeIgniteStore +import org.finos.vuu.test.FakeDataSource import org.scalatest.BeforeAndAfterEach import org.scalatest.featurespec.AnyFeatureSpec class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterEach { - val igniteStore: FakeIgniteStore[SchemaTestData] = new FakeIgniteStore[SchemaTestData] - + private val fakeDataSource: FakeDataSource[SchemaTestData] = new FakeDataSource[SchemaTestData] protected var queryName: String = "myQuery" override def beforeEach(): Unit = { @@ -17,23 +16,21 @@ class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterE } protected def getDataAndUpdateTable(table: DataTable, schemaMapper: SchemaMapper, queryName: String): Unit = { - val result = getIgniteQueryResult(queryName) + val result = getQueryResult(queryName) val rows = mapToRow(result, schemaMapper, table.getTableDef) rows.foreach(row => table.processUpdate(row.key, row, 0)) } - //todo make it not specific to ignite? type the results more? - protected def givenIgniteSqlFieldQueryReturns(queryName: String, results: List[List[Any]]): Unit = { - igniteStore.setUpSqlFieldsQuery(queryName, results) + protected def givenQueryReturns(queryName: String, results: List[List[Any]]): Unit = { + fakeDataSource.setUpResultAsListOfValues(queryName, results) } - protected def getIgniteQueryResult(queryName: String): List[List[Any]] = { - igniteStore.getSqlFieldsQuery(queryName) + private def getQueryResult(queryName: String): List[List[Any]] = { + fakeDataSource.getAsListOfValues(queryName) .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) } - - def mapToRow(resultData: List[List[Any]], schemaMapper: SchemaMapper, tableDef: TableDef): Seq[RowWithData] = { + private def mapToRow(resultData: List[List[Any]], schemaMapper: SchemaMapper, tableDef: TableDef): Seq[RowWithData] = { // map to entity object - as order of values are relevant to how the query schema was defined //todo two options, is direct to row map better if query result returns values? @@ -51,12 +48,21 @@ class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterE RowWithData(keyValue, rowMap) }) } + + protected def createExternalEntitySchema: ExternalEntitySchema = + ExternalEntitySchemaBuilder() + .withEntity(classOf[SchemaTestData]) + .withIndex("ID_INDEX", List("id")) + .build() + + //todo try more scenarios // - virtual table - include filter, type ahead - // - error scenarios fetching data or converting types - review api for returning error + // - example of generating table Def from an class rather than using column builder? maybe more of a unit teest/example for table def? //todo to respect the QueryEntity order of fields, if it is different from order of fields on the entity class, should be generated using that? + //error scenarios fetching data or converting types - review api for returning error //todo when query result has less number of fields than table column //todo when fields and column diff order but no mapping specified - assert on error message being useful From e58bc532408ee5740bc6f40b065580f99a279bcf Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Thu, 16 May 2024 15:37:59 +0100 Subject: [PATCH 11/19] #1318 introduce function to create table columns from external schema --- .../main/scala/org/finos/vuu/api/ColumnBuilder.scala | 3 +-- .../main/scala/org/finos/vuu/core/table/Column.scala | 12 ++++++++++-- .../vuu/util/schema/SchemaMapperFunctionalTest.scala | 7 ++----- .../util/schema/SchemaMapperFunctionalTestBase.scala | 2 -- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala index 534835d31..b817b9b9e 100644 --- a/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala +++ b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala @@ -26,9 +26,8 @@ class ColumnBuilder { def addLong(columnName: String): ColumnBuilder = { columns += (columnName + ":Long") this - - } + def build(): Array[Column] = Columns.fromNames(columns.result()) } diff --git a/vuu/src/main/scala/org/finos/vuu/core/table/Column.scala b/vuu/src/main/scala/org/finos/vuu/core/table/Column.scala index d5a0ca9e3..50427f068 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/table/Column.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/table/Column.scala @@ -3,6 +3,7 @@ package org.finos.vuu.core.table import com.typesafe.scalalogging.StrictLogging import org.finos.vuu.api.TableDef import org.finos.vuu.core.table.column.CalculatedColumnClause +import org.finos.vuu.util.schema.ExternalEntitySchema import org.finos.vuu.util.types.{DefaultTypeConverters, TypeConverterContainerBuilder} import scala.util.Try @@ -86,8 +87,6 @@ object Columns { table.columns.filter(c => aliased.contains(c.name)) map (c => new AliasedJoinColumn(aliased(c.name), c.index, c.dataType, table, c).asInstanceOf[Column]) } - //def calculated(name: String, definition: String): Array[Column] = ??? - def allFromExcept(table: TableDef, excludeColumns: String*): Array[Column] = { val excluded = excludeColumns.map(s => s -> 1).toMap @@ -95,6 +94,15 @@ object Columns { table.columns.filterNot(c => excluded.contains(c.name)).map(c => new JoinColumn(c.name, c.index, c.dataType, table, c)) } + /** + * Create columns that use same name, type, order as the external entity fields + * */ + + def fromExternalSchema(externalSchema: ExternalEntitySchema): Array[Column] = { + externalSchema.fields.map(field => SimpleColumn(field.name, field.index, field.dataType)) + .toArray + } + } trait Column { diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala index 7cd40681f..4023b4762 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala @@ -1,6 +1,7 @@ package org.finos.vuu.util.schema import org.finos.vuu.api.{ColumnBuilder, TableDef} +import org.finos.vuu.core.table.Columns import org.finos.vuu.test.FakeInMemoryTable class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { @@ -12,11 +13,7 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { val tableDef = TableDef( name = "MyExampleTable", keyField = "id", - columns = new ColumnBuilder() - .addString("id") - .addInt("clientId") - .addDouble("notionalValue") - .build() + columns = Columns.fromExternalSchema(externalEntitySchema) ) val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) .build() diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala index 262fe1c63..9d1220551 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala @@ -59,8 +59,6 @@ class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterE //todo try more scenarios // - virtual table - include filter, type ahead - // - example of generating table Def from an class rather than using column builder? maybe more of a unit teest/example for table def? - //todo to respect the QueryEntity order of fields, if it is different from order of fields on the entity class, should be generated using that? //error scenarios fetching data or converting types - review api for returning error //todo when query result has less number of fields than table column From 8b15000627aed216c38d225a7dc0398dc3798fc5 Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Thu, 16 May 2024 17:41:16 +0100 Subject: [PATCH 12/19] #1318 adding virtualized table plugin table dependency in ignite plugin to be able to add schema mapper functional tests --- plugin/ignite-plugin/pom.xml | 6 ++ .../ignite/SchemaMapperFunctionalTest.scala | 80 +++++++++++++++++++ .../schema/SchemaMapperFunctionalTest.scala | 6 +- .../SchemaMapperFunctionalTestBase.scala | 2 +- 4 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/SchemaMapperFunctionalTest.scala diff --git a/plugin/ignite-plugin/pom.xml b/plugin/ignite-plugin/pom.xml index 427277ace..c740c4c65 100644 --- a/plugin/ignite-plugin/pom.xml +++ b/plugin/ignite-plugin/pom.xml @@ -57,6 +57,12 @@ 0.9.65-SNAPSHOT + + org.finos.vuu.plugin + virtualized-table-plugin + 0.9.65-SNAPSHOT + + org.scala-lang scala-library diff --git a/plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/SchemaMapperFunctionalTest.scala b/plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/SchemaMapperFunctionalTest.scala new file mode 100644 index 000000000..aac258e2a --- /dev/null +++ b/plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/SchemaMapperFunctionalTest.scala @@ -0,0 +1,80 @@ +package org.finos.vuu.feature.ignite + +import org.finos.vuu.core.sort.SortDirection +import org.finos.vuu.core.table.{ColumnValueProvider, Columns} +import org.finos.vuu.net.FilterSpec +import org.finos.vuu.plugin.virtualized.api.VirtualizedSessionTableDef +import org.finos.vuu.provider.VirtualizedProvider +import org.finos.vuu.test.{FakeDataSource, FakeInMemoryTable} +import org.finos.vuu.util.schema.{ExternalEntitySchema, SchemaMapperBuilder, SchemaMapperFunctionalTestBase, SchemaTestData} +import org.finos.vuu.viewport.ViewPort + +class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { + + Feature("Filter data in virtualised table using schema mapper") { + Scenario("When table columns and entity fields has same type") { + + val externalEntitySchema: ExternalEntitySchema = createExternalEntitySchema + val tableDef = VirtualizedSessionTableDef( + name = "MyExampleVirtualTable", + keyField = "id", + columns = Columns.fromExternalSchema(externalEntitySchema) + ) + val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + .build() + val table = new FakeInMemoryTable("SchemaMapTest", tableDef) + + + //trigger type ahead and get data from data source, replicate returning it as table column type + givenQueryReturns("unique", List( + List("testId1", 5, 10.5), + List("testId2", 6, 11.5), + List("testId3", 5, 12.5), + )) + val dataProvider = new TestVirtualProvider(fakeDataSource) + val columnValueProvider = dataProvider.asInstanceOf[ColumnValueProvider] + columnValueProvider.getUniqueValues("clientId") + //todo need to convert to column type before sending results to ui if different from schema type + + //get user specified filter spec, run through antler parser and all the way to datasource query and return result + + + //todo does these tests need to use real ignite? + + val filterAndSortSpecToSql = FilterAndSortSpecToSql(schemaMapper) + + + val filterSpec = FilterSpec("orderId > 1 and ric starts \"ABC\"") + val sortSpec = Map("price" -> SortDirection.Ascending) + + givenQueryReturns("filtered", List( + List("testId1", 5, 10.5), + List("testId2", 6, 11.5), + List("testId3", 5, 12.5), + )) + + fakeDataSource.getAsListOfValues("filtered") + } + } +} + + +class TestVirtualProvider(fakeDataSource:FakeDataSource[SchemaTestData]) extends VirtualizedProvider { + override def runOnce(viewPort: ViewPort): Unit = ??? + + override def getUniqueValues(columnName: String): Array[String] = fakeDataSource.getAsListOfValues("unique") + + override def getUniqueValuesStartingWith(columnName: String, starts: String): Array[String] = ??? + + override def subscribe(key: String): Unit = ??? + + override def doStart(): Unit = ??? + + override def doStop(): Unit = ??? + + override def doInitialize(): Unit = ??? + + override def doDestroy(): Unit = ??? + + override val lifecycleId: String = ??? +} diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala index 4023b4762..06fd18efc 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala @@ -9,8 +9,8 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { Feature("Update in memory table using schema mapper") { Scenario("When table columns and entity fields match exactly") { - val externalEntitySchema: ExternalEntitySchema = createExternalEntitySchema - val tableDef = TableDef( + val externalEntitySchema: ExternalEntitySchema = createExternalEntitySchema + val tableDef = TableDef( name = "MyExampleTable", keyField = "id", columns = Columns.fromExternalSchema(externalEntitySchema) @@ -118,4 +118,4 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { assert(existingTableRows.head.get("notionalValue") == 10.5) } } -} +} \ No newline at end of file diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala index 9d1220551..f70e67515 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala @@ -8,7 +8,7 @@ import org.scalatest.featurespec.AnyFeatureSpec class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterEach { - private val fakeDataSource: FakeDataSource[SchemaTestData] = new FakeDataSource[SchemaTestData] + protected val fakeDataSource: FakeDataSource[SchemaTestData] = new FakeDataSource[SchemaTestData] protected var queryName: String = "myQuery" override def beforeEach(): Unit = { From 18062b5e96cf61f156e5bcfc9038284edea6d1f6 Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Fri, 17 May 2024 17:00:38 +0100 Subject: [PATCH 13/19] #1318 adding additional type for column builder --- .../main/scala/org/finos/vuu/api/ColumnBuilder.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala index b817b9b9e..8ffc61f88 100644 --- a/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala +++ b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala @@ -27,6 +27,16 @@ class ColumnBuilder { columns += (columnName + ":Long") this } + + def addBoolean(columnName: String): ColumnBuilder = { + columns += (columnName + ":Boolean") + this + } + + def addChar(columnName: String): ColumnBuilder = { + columns += (columnName + ":Char") + this + } def build(): Array[Column] = Columns.fromNames(columns.result()) } From f099eb739a77672a3d1d2b7d98469ea44d96a802 Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Fri, 17 May 2024 18:24:31 +0100 Subject: [PATCH 14/19] #1318 adding additional test scenario to be implemented --- .../vuu/util/schema/SchemaMapperFunctionalTest.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala index 06fd18efc..f3d3471d7 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala @@ -118,4 +118,12 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { assert(existingTableRows.head.get("notionalValue") == 10.5) } } + + Scenario("When table has columns are different type from fields on external entity") {} + Scenario("When query result has less number of fields than table columns return useful error") {} + Scenario("When column and field order does not match but no fields map defined on schema return useful error") {} + Scenario("When getting data from source fails return useful error") {} + Scenario("When getting casting data from source to table column type fails return useful error") {} + + } \ No newline at end of file From 85236d3d59010eec7eed4e2377d93fef8de1e6f7 Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Fri, 17 May 2024 18:25:03 +0100 Subject: [PATCH 15/19] #1318 adding java usecase tests for schema mapper --- .../org/finos/vuu/util/SchemaJavaExample.java | 90 ------------ .../finos/vuu/util/SchemaJavaTestData.java | 1 + .../util/SchemaMapperJavaFunctionalTest.java | 134 ++++++++++++++++++ .../SchemaMapperFunctionalTestBase.scala | 8 +- 4 files changed, 140 insertions(+), 93 deletions(-) delete mode 100644 vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java create mode 100644 vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java deleted file mode 100644 index 11bf88721..000000000 --- a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaExample.java +++ /dev/null @@ -1,90 +0,0 @@ -package org.finos.vuu.util; - -import org.finos.vuu.api.ColumnBuilder; -import org.finos.vuu.api.TableDef; -import org.finos.vuu.core.table.RowWithData; -import org.finos.vuu.test.FakeDataSource; -import org.finos.vuu.test.FakeInMemoryTable; -import org.finos.vuu.util.schema.ExternalEntitySchemaBuilder; -import org.finos.vuu.util.schema.SchemaMapperBuilder; -import scala.jdk.javaapi.OptionConverters; - -import java.util.List; - -import static org.finos.vuu.util.ScalaCollectionConverter.*; - -public class SchemaJavaExample { - - public static void main(String[] args) throws Exception { - - doSomething(); - } - public static void doSomething() throws Exception { - - //create table def - var tableDef = TableDef.apply( - "MyExampleTable", - "Id", - new ColumnBuilder() - .addString("Id") - .addDouble("NotionalValue") - .build(), - toScalaSeq(List.of()) - ); - - - //create entity schema - var externalEntitySchema = ExternalEntitySchemaBuilder.apply() - .withEntity(SchemaJavaTestData.class) - .withIndex("ID_INDEX", toScala( List.of("Id"))) - .build(); - - //create schema mapper - var schemaMapper = SchemaMapperBuilder.apply(externalEntitySchema, tableDef.columns()) - //.withFieldsMap(columnNameByExternalField) - .build(); - - - - //get data from ignite as list of values - var queryName = "myQuery"; - var igniteStore = new FakeDataSource(); - igniteStore.setUpResultAsListOfValues( - queryName, - ScalaList.of(ScalaList.of("id1", 10.5)) - ); - - //todo should use fake java store? - List> result = - OptionConverters.toJava(igniteStore.getAsListOfValues(queryName)) - .map(listOfLists -> toJava(listOfLists.map(ScalaCollectionConverter::toJava).toList())) - .orElseThrow(()-> new Exception("query does not exist in store. make sure it is setup")); - - // map to entity object and then to row - - //map to row directly - var tableRowMap = result.stream() - .map(rowData -> - schemaMapper.toInternalRowMap(toScala(rowData) - )); - //.map(rowMap -> toJava(rowMap)); - - //map to tablerow - var keyFieldName = tableDef.keyField(); - var tableRows = tableRowMap.map(rowMap -> { - var keyOptional = OptionConverters.toJava(rowMap.get(keyFieldName)); - var key = keyOptional.orElseThrow(); - return new RowWithData(key.toString(), rowMap); - }); - - //update table with table row? - var table = new FakeInMemoryTable("SchemaMapJavaTest", tableDef); - tableRows.forEach(row -> table.processUpdate(row.key(), row, 0)); //todo use clock now - - //assert on reading the table row - is that possible or need to use mock table with table interface - var existingRows = table.pullAllRows(); - } - -} - - diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java index 250c3c475..d5a32ff09 100644 --- a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java @@ -2,5 +2,6 @@ public class SchemaJavaTestData { public String Id; + public int ClientId; public double NotionalValue; } diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java b/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java new file mode 100644 index 000000000..421c804f9 --- /dev/null +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java @@ -0,0 +1,134 @@ +package org.finos.vuu.util; + + +import org.finos.toolbox.time.Clock; +import org.finos.toolbox.time.TestFriendlyClock; +import org.finos.vuu.api.ColumnBuilder; +import org.finos.vuu.api.TableDef; +import org.finos.vuu.core.table.Columns; +import org.finos.vuu.core.table.RowWithData; +import org.finos.vuu.test.FakeDataSource; +import org.finos.vuu.test.FakeInMemoryTable; +import org.finos.vuu.util.schema.ExternalEntitySchemaBuilder; +import org.finos.vuu.util.schema.SchemaMapper; +import org.finos.vuu.util.schema.SchemaMapperBuilder; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import scala.jdk.javaapi.OptionConverters; + +import java.util.List; +import java.util.Map; + +import static org.finos.vuu.util.ScalaCollectionConverter.*; +import static org.junit.Assert.assertEquals; + + +@RunWith(Enclosed.class) +public class SchemaMapperJavaFunctionalTest { + + private static String queryName = "myQuery"; + private static FakeDataSource dataSource = new FakeDataSource<>(); + private static Clock clock = new TestFriendlyClock(10001L); + + @Before + public void setUp() { + queryName += java.util.UUID.randomUUID().toString(); // unique query name for each test run + } + + public static class UpdateInMemoryTable { + @Test + public void when_table_columns_and_entity_fields_match_exactly() throws Exception { + + var externalEntitySchema = ExternalEntitySchemaBuilder.apply() + .withEntity(SchemaJavaTestData.class) + .withIndex("ID_INDEX", toScala(List.of("Id"))) + .build(); + var tableDef = TableDef.apply( + "MyJavaExampleTable", + "Id", + Columns.fromExternalSchema(externalEntitySchema), + toScalaSeq(List.of()) + ); + var schemaMapper = SchemaMapperBuilder.apply(externalEntitySchema, tableDef.columns()) + .build(); + var table = new FakeInMemoryTable("SchemaMapJavaTest", tableDef); + dataSource.setUpResultAsListOfValues( + queryName, + ScalaList.of(ScalaList.of("testId1", 5, 10.5)) + ); + + getDataAndUpdateTable(queryName, schemaMapper, table); + + var existingRows = toJava(table.pullAllRows()); + assertEquals(existingRows.size(), 1); + assertEquals(existingRows.get(0).get("Id"), "testId1"); + assertEquals(existingRows.get(0).get("ClientId"), 5); + assertEquals(existingRows.get(0).get("NotionalValue"), 10.5); + } + + @Test + public void when_table_columns_and_entity_fields_does_not_match_exactly() throws Exception { + + var externalEntitySchema = ExternalEntitySchemaBuilder.apply() + .withEntity(SchemaJavaTestData.class) + .withIndex("ID_INDEX", toScala(List.of("Id"))) + .build(); + var tableDef = TableDef.apply( + "MyJavaExampleTable", + "Id", + new ColumnBuilder() + .addDouble("SomeOtherName") + .addString("Id") + .addInt("ClientId") + .build(), + toScalaSeq(List.of()) + ); + var schemaMapper = SchemaMapperBuilder.apply(externalEntitySchema, tableDef.columns()) + .withFieldsMap( + toScala(Map.of("Id", "Id", + "ClientId", "ClientId", + "NotionalValue", "SomeOtherName" + )) + ) + .build(); + var table = new FakeInMemoryTable("SchemaMapJavaTest", tableDef); + dataSource.setUpResultAsListOfValues( + queryName, + ScalaList.of(ScalaList.of("testId1", 5, 10.5)) + ); + + getDataAndUpdateTable(queryName, schemaMapper, table); + + var existingRows = toJava(table.pullAllRows()); + assertEquals(existingRows.size(), 1); + assertEquals(existingRows.get(0).get("Id"), "testId1"); + assertEquals(existingRows.get(0).get("ClientId"), 5); + assertEquals(existingRows.get(0).get("SomeOtherName"), 10.5); + + } + } + + private static void getDataAndUpdateTable(String queryName, SchemaMapper schemaMapper, FakeInMemoryTable table) throws Exception { + //todo should use fake java store which is more likely usecase and avoid all the type conversions? + var keyFieldName = table.getTableDef().keyField(); + getQueryResult(queryName).stream() + .map(valueList -> mapToRows(schemaMapper, valueList, keyFieldName)) + .forEach(row -> table.processUpdate(row.key(), row, clock.now())); + } + + private static RowWithData mapToRows(SchemaMapper schemaMapper, List valueList, String keyFieldName) { + var rowMap = schemaMapper.toInternalRowMap(toScala(valueList)); + var keyOptional = OptionConverters.toJava(rowMap.get(keyFieldName)); + var key = keyOptional.orElseThrow(); + return new RowWithData(key.toString(), rowMap); + } + + private static List> getQueryResult(String queryName) throws Exception { + var result = OptionConverters.toJava(dataSource.getAsListOfValues(queryName)) + .map(listOfLists -> toJava(listOfLists.map(ScalaCollectionConverter::toJava).toList())) + .orElseThrow(() -> new Exception("Query does not exist in store. make sure it is setup")); + return result; + } +} diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala index f70e67515..fc4a21c87 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala @@ -1,5 +1,6 @@ package org.finos.vuu.util.schema +import org.finos.toolbox.time.{Clock, TestFriendlyClock} import org.finos.vuu.api.TableDef import org.finos.vuu.core.table.{DataTable, RowWithData} import org.finos.vuu.test.FakeDataSource @@ -10,6 +11,7 @@ class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterE protected val fakeDataSource: FakeDataSource[SchemaTestData] = new FakeDataSource[SchemaTestData] protected var queryName: String = "myQuery" + private val clock = new TestFriendlyClock(10001L) override def beforeEach(): Unit = { queryName += java.util.UUID.randomUUID.toString // unique query name for each test run @@ -17,8 +19,8 @@ class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterE protected def getDataAndUpdateTable(table: DataTable, schemaMapper: SchemaMapper, queryName: String): Unit = { val result = getQueryResult(queryName) - val rows = mapToRow(result, schemaMapper, table.getTableDef) - rows.foreach(row => table.processUpdate(row.key, row, 0)) + val rows = mapToRows(result, schemaMapper, table.getTableDef) + rows.foreach(row => table.processUpdate(row.key, row, clock.now())) } protected def givenQueryReturns(queryName: String, results: List[List[Any]]): Unit = { @@ -30,7 +32,7 @@ class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterE .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) } - private def mapToRow(resultData: List[List[Any]], schemaMapper: SchemaMapper, tableDef: TableDef): Seq[RowWithData] = { + private def mapToRows(resultData: List[List[Any]], schemaMapper: SchemaMapper, tableDef: TableDef): Seq[RowWithData] = { // map to entity object - as order of values are relevant to how the query schema was defined //todo two options, is direct to row map better if query result returns values? From dbdd6abfd7a4c1b1e7d1093439b3cc962877ecb0 Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Thu, 30 May 2024 15:23:04 +0100 Subject: [PATCH 16/19] #1318 test cleanup --- .../util/SchemaMapperJavaFunctionalTest.java | 4 +- .../schema/SchemaMapperFunctionalTest.scala | 14 ++--- .../SchemaMapperFunctionalTestBase.scala | 62 ++++--------------- 3 files changed, 21 insertions(+), 59 deletions(-) diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java b/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java index 421c804f9..78c0185e0 100644 --- a/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java @@ -114,11 +114,11 @@ private static void getDataAndUpdateTable(String queryName, SchemaMapper schemaM //todo should use fake java store which is more likely usecase and avoid all the type conversions? var keyFieldName = table.getTableDef().keyField(); getQueryResult(queryName).stream() - .map(valueList -> mapToRows(schemaMapper, valueList, keyFieldName)) + .map(rowValues -> mapToRow(schemaMapper, rowValues, keyFieldName)) .forEach(row -> table.processUpdate(row.key(), row, clock.now())); } - private static RowWithData mapToRows(SchemaMapper schemaMapper, List valueList, String keyFieldName) { + private static RowWithData mapToRow(SchemaMapper schemaMapper, List valueList, String keyFieldName) { var rowMap = schemaMapper.toInternalRowMap(toScala(valueList)); var keyOptional = OptionConverters.toJava(rowMap.get(keyFieldName)); var key = keyOptional.orElseThrow(); diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala index f3d3471d7..484eb50df 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala @@ -117,13 +117,11 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { assert(existingTableRows.head.get("clientId") == 5) assert(existingTableRows.head.get("notionalValue") == 10.5) } - } - - Scenario("When table has columns are different type from fields on external entity") {} - Scenario("When query result has less number of fields than table columns return useful error") {} - Scenario("When column and field order does not match but no fields map defined on schema return useful error") {} - Scenario("When getting data from source fails return useful error") {} - Scenario("When getting casting data from source to table column type fails return useful error") {} - + Scenario("When table has columns with different type from fields on external entity") {} + Scenario("When query result has less number of fields than table columns return useful error") {} + Scenario("When column and field order does not match but no fields map defined on schema return useful error") {} + Scenario("When getting data from source fails return useful error") {} + Scenario("When getting casting data from source to table column type fails return useful error") {} + } } \ No newline at end of file diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala index fc4a21c87..424cff27c 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala @@ -1,7 +1,6 @@ package org.finos.vuu.util.schema -import org.finos.toolbox.time.{Clock, TestFriendlyClock} -import org.finos.vuu.api.TableDef +import org.finos.toolbox.time.TestFriendlyClock import org.finos.vuu.core.table.{DataTable, RowWithData} import org.finos.vuu.test.FakeDataSource import org.scalatest.BeforeAndAfterEach @@ -18,9 +17,17 @@ class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterE } protected def getDataAndUpdateTable(table: DataTable, schemaMapper: SchemaMapper, queryName: String): Unit = { - val result = getQueryResult(queryName) - val rows = mapToRows(result, schemaMapper, table.getTableDef) - rows.foreach(row => table.processUpdate(row.key, row, clock.now())) + val keyFieldName = table.getTableDef.keyField + + getQueryResult(queryName) + .map(rowValues => mapToRow(schemaMapper, keyFieldName, rowValues)) + .foreach(row => table.processUpdate(row.key, row, clock.now())) + } + + private def mapToRow(schemaMapper: SchemaMapper, keyFieldName: String, rowValues: List[Any]) = { + val rowMap = schemaMapper.toInternalRowMap(rowValues) + val keyValue = rowMap(keyFieldName).toString + RowWithData(keyValue, rowMap) } protected def givenQueryReturns(queryName: String, results: List[List[Any]]): Unit = { @@ -32,54 +39,11 @@ class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterE .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) } - private def mapToRows(resultData: List[List[Any]], schemaMapper: SchemaMapper, tableDef: TableDef): Seq[RowWithData] = { - - // map to entity object - as order of values are relevant to how the query schema was defined - //todo two options, is direct to row map better if query result returns values? - // val tableRowMap1 = resultData - // .map(rowData => mapToEntity(rowData)) - // .map(externalEntity => schemaMapper.toInternalRowMap(externalEntity)) - - val tableRowMap2: Seq[Map[String, Any]] = - resultData.map(rowData => schemaMapper.toInternalRowMap(rowData)) - - //map to tablerow - val keyFieldName = tableDef.keyField - tableRowMap2.map(rowMap => { - val keyValue = rowMap(keyFieldName).toString - RowWithData(keyValue, rowMap) - }) - } - protected def createExternalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() .withEntity(classOf[SchemaTestData]) .withIndex("ID_INDEX", List("id")) .build() - - - //todo try more scenarios - // - virtual table - include filter, type ahead - - //todo to respect the QueryEntity order of fields, if it is different from order of fields on the entity class, should be generated using that? - //error scenarios fetching data or converting types - review api for returning error - //todo when query result has less number of fields than table column - //todo when fields and column diff order but no mapping specified - assert on error message being useful - - - // //todo different for java - // private def mapToEntity(rowData: List[Any]): SchemaTestData = - // getListToObjectConverter[SchemaTestData](SchemaTestData)(rowData) - } -//copy of one in org.finos.vuu.example.ignite.utils -////todo if not going to use or move to common place -//object getListToObjectConverter { -// def apply[ReturnType](obj: Object): List[_] => ReturnType = { -// val converter = obj.getClass.getMethods.find(x => x.getName == "apply" && x.isBridge).get -// values => converter.invoke(obj, values.map(_.asInstanceOf[AnyRef]): _*).asInstanceOf[ReturnType] -// } -//} - -case class SchemaTestData(id :String, clientId:Int, notionalValue: Double) {} \ No newline at end of file +case class SchemaTestData(id: String, clientId: Int, notionalValue: Double) {} \ No newline at end of file From a47ab184483158647406e4c52650a5de9b091113 Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Thu, 30 May 2024 15:50:58 +0100 Subject: [PATCH 17/19] #1318 SchemaMapper functional test for virtualised table --- .../ignite/SchemaMapperFunctionalTest.scala | 35 ++++++++++--------- .../org/finos/vuu/test/FakeDataSource.scala | 13 +++++++ .../SchemaMapperFunctionalTestBase.scala | 4 +++ 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/SchemaMapperFunctionalTest.scala b/plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/SchemaMapperFunctionalTest.scala index aac258e2a..340344316 100644 --- a/plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/SchemaMapperFunctionalTest.scala +++ b/plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/SchemaMapperFunctionalTest.scala @@ -24,37 +24,35 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { .build() val table = new FakeInMemoryTable("SchemaMapTest", tableDef) + //simulate using typeahead + givenColumnQueryReturns("unique", "clientId", Array("5","6")) - //trigger type ahead and get data from data source, replicate returning it as table column type - givenQueryReturns("unique", List( - List("testId1", 5, 10.5), - List("testId2", 6, 11.5), - List("testId3", 5, 12.5), - )) val dataProvider = new TestVirtualProvider(fakeDataSource) val columnValueProvider = dataProvider.asInstanceOf[ColumnValueProvider] columnValueProvider.getUniqueValues("clientId") - //todo need to convert to column type before sending results to ui if different from schema type - //get user specified filter spec, run through antler parser and all the way to datasource query and return result + //todo assert on the result returned for typeahead - - //todo does these tests need to use real ignite? + //simulate using user entered filter and sort to the data query + val filterSpec = FilterSpec("orderId > 1 and ric starts \"ABC\"") + val sortSpec = Map("price" -> SortDirection.Ascending) val filterAndSortSpecToSql = FilterAndSortSpecToSql(schemaMapper) + filterAndSortSpecToSql.sortToSql(sortSpec) + filterAndSortSpecToSql.filterToSql(filterSpec) + //todo assert that correct sql query is created - should use real ignite or assert on expected sql query? - val filterSpec = FilterSpec("orderId > 1 and ric starts \"ABC\"") - val sortSpec = Map("price" -> SortDirection.Ascending) - + //todo test once query is returned it can be mapped appropriate to table rows & assert on what exist in table givenQueryReturns("filtered", List( List("testId1", 5, 10.5), List("testId2", 6, 11.5), List("testId3", 5, 12.5), )) - fakeDataSource.getAsListOfValues("filtered") } + + Scenario("When table columns and entity fields has different type"){} } } @@ -62,7 +60,7 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { class TestVirtualProvider(fakeDataSource:FakeDataSource[SchemaTestData]) extends VirtualizedProvider { override def runOnce(viewPort: ViewPort): Unit = ??? - override def getUniqueValues(columnName: String): Array[String] = fakeDataSource.getAsListOfValues("unique") + override def getUniqueValues(columnName: String): Array[String] = getColumnQueryResult("unique", columnName) override def getUniqueValuesStartingWith(columnName: String, starts: String): Array[String] = ??? @@ -76,5 +74,10 @@ class TestVirtualProvider(fakeDataSource:FakeDataSource[SchemaTestData]) extends override def doDestroy(): Unit = ??? - override val lifecycleId: String = ??? + override val lifecycleId: String = "SchemaMapperFunctionalTest" + + private def getColumnQueryResult(queryName: String, columnName:String): Array[String] = { + fakeDataSource.getColumnValues(queryName, columnName) + .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) + } } diff --git a/vuu/src/test/scala/org/finos/vuu/test/FakeDataSource.scala b/vuu/src/test/scala/org/finos/vuu/test/FakeDataSource.scala index f448f5e50..ae462b9aa 100644 --- a/vuu/src/test/scala/org/finos/vuu/test/FakeDataSource.scala +++ b/vuu/src/test/scala/org/finos/vuu/test/FakeDataSource.scala @@ -3,6 +3,7 @@ class FakeDataSource[T]{ private val queryToEntityResultMap = scala.collection.mutable.HashMap.empty[String, List[T]] private val queryToValuesResultMap = scala.collection.mutable.HashMap.empty[String, List[List[Any]]] + private val queryToColumnResultMap = scala.collection.mutable.HashMap.empty[String, Array[String]] def setUpResultAsObjects(queryName:String, queryResult:List[T]): Unit = { queryToEntityResultMap += (queryName -> queryResult) @@ -19,4 +20,16 @@ class FakeDataSource[T]{ def getAsListOfValues(queryName: String): Option[List[List[Any]]] = { queryToValuesResultMap.get(queryName) } + + def setUpResultGivenColumn(queryName:String, columnName:String, queryResult:Array[String]): Unit = { + queryToColumnResultMap += (getQueryColumnName(queryName, columnName) -> queryResult) + } + + def getColumnValues(queryName: String, columnName:String): Option[Array[String]] = { + queryToColumnResultMap.get(getQueryColumnName(queryName, columnName)) + } + + private def getQueryColumnName(queryName: String, columnName: String) = { + queryName + ":" + columnName + } } \ No newline at end of file diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala index 424cff27c..b6611595e 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala @@ -39,6 +39,10 @@ class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterE .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) } + protected def givenColumnQueryReturns(queryName: String, columnName:String, results: Array[String]): Unit = { + fakeDataSource.setUpResultGivenColumn(queryName, columnName, results) + } + protected def createExternalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() .withEntity(classOf[SchemaTestData]) From 0636990e613f5ef56ad9f30ea5084bb5c4739a9d Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Fri, 31 May 2024 15:45:59 +0100 Subject: [PATCH 18/19] #1318 clean up based on review comments --- .../scala/org/finos/vuu/core/table/Column.scala | 1 - .../java/org/finos/vuu/util/ScalaList.java | 0 .../org/finos/vuu/test/FakeInMemoryTable.scala | 13 ------------- 3 files changed, 14 deletions(-) rename vuu/src/{main => test}/java/org/finos/vuu/util/ScalaList.java (100%) diff --git a/vuu/src/main/scala/org/finos/vuu/core/table/Column.scala b/vuu/src/main/scala/org/finos/vuu/core/table/Column.scala index 50427f068..7671fb430 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/table/Column.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/table/Column.scala @@ -97,7 +97,6 @@ object Columns { /** * Create columns that use same name, type, order as the external entity fields * */ - def fromExternalSchema(externalSchema: ExternalEntitySchema): Array[Column] = { externalSchema.fields.map(field => SimpleColumn(field.name, field.index, field.dataType)) .toArray diff --git a/vuu/src/main/java/org/finos/vuu/util/ScalaList.java b/vuu/src/test/java/org/finos/vuu/util/ScalaList.java similarity index 100% rename from vuu/src/main/java/org/finos/vuu/util/ScalaList.java rename to vuu/src/test/java/org/finos/vuu/util/ScalaList.java diff --git a/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala b/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala index 7898fdb27..5df637338 100644 --- a/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala +++ b/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala @@ -32,17 +32,8 @@ class FakeInMemoryTable(val instanceName: String, val tableDef: TableDef) extend override def processDelete(rowKey: String): Unit = ??? - /** - * notify listeners explicit when a rowKey changes - */ override def notifyListeners(rowKey: String, isDelete: Boolean): Unit = ??? - /** - * Link table name is the name of the underlying table that we can link to. - * In a session table this would be the underlying table. - * - * @return - */ override def linkableName: String = ??? override def readRow(key: String, columns: List[String], processor: RowProcessor): Unit = ??? @@ -51,10 +42,6 @@ class FakeInMemoryTable(val instanceName: String, val tableDef: TableDef) extend override def pullRow(key: String, columns: ViewPortColumns): RowData = ??? - /** - * Note the below call should only be used for testing. It filters the contents of maps by the expected viewPortColumns. - * In practice we never need to do this at runtime. - */ override def pullRowFiltered(key: String, columns: ViewPortColumns): RowData = ??? override def pullRowAsArray(key: String, columns: ViewPortColumns): Array[Any] = ??? From 587b0466ef030b4d42232f618b9f476311bea81c Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Tue, 4 Jun 2024 16:49:54 +0100 Subject: [PATCH 19/19] #1318 schema mapper functional test for when table column type is different from field type --- .../util/SchemaMapperJavaFunctionalTest.java | 53 +++++++++++++++++-- .../schema/SchemaMapperFunctionalTest.scala | 43 ++++++++++++++- 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java b/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java index 78c0185e0..0afc51694 100644 --- a/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java @@ -12,12 +12,15 @@ import org.finos.vuu.util.schema.ExternalEntitySchemaBuilder; import org.finos.vuu.util.schema.SchemaMapper; import org.finos.vuu.util.schema.SchemaMapperBuilder; +import org.finos.vuu.util.types.TypeConverter; +import org.finos.vuu.util.types.TypeConverterContainerBuilder; import org.junit.Before; import org.junit.Test; import org.junit.experimental.runners.Enclosed; import org.junit.runner.RunWith; import scala.jdk.javaapi.OptionConverters; +import java.math.BigDecimal; import java.util.List; import java.util.Map; @@ -29,8 +32,8 @@ public class SchemaMapperJavaFunctionalTest { private static String queryName = "myQuery"; - private static FakeDataSource dataSource = new FakeDataSource<>(); - private static Clock clock = new TestFriendlyClock(10001L); + private static final FakeDataSource dataSource = new FakeDataSource<>(); + private static final Clock clock = new TestFriendlyClock(10001L); @Before public void setUp() { @@ -108,6 +111,49 @@ public void when_table_columns_and_entity_fields_does_not_match_exactly() throws assertEquals(existingRows.get(0).get("SomeOtherName"), 10.5); } + + @Test + public void when_table_columns_and_entity_fields_has_different_types() throws Exception { + + var externalEntitySchema = ExternalEntitySchemaBuilder.apply() + .withField("Id", Integer.class) + .withField("decimalValue", BigDecimal.class) + .withIndex("ID_INDEX", toScala(List.of("Id"))) + .build(); + var tableDef = TableDef.apply( + "MyJavaExampleTable", + "Id", + new ColumnBuilder() + .addString("Id") + .addDouble("doubleValue") + .build(), + toScalaSeq(List.of()) + ); + var typeConverterContainer = TypeConverterContainerBuilder.apply() + .withConverter(TypeConverter.apply(BigDecimal.class, Double.class, BigDecimal::doubleValue)) + .withConverter(TypeConverter.apply(Double.class, BigDecimal.class, v -> new BigDecimal(v.toString()))) + .build(); + var schemaMapper = SchemaMapperBuilder.apply(externalEntitySchema, tableDef.columns()) + .withFieldsMap( + toScala(Map.of("Id", "Id", + "decimalValue","doubleValue" + )) + ) + .withTypeConverters(typeConverterContainer) + .build(); + var table = new FakeInMemoryTable("SchemaMapJavaTest", tableDef); + dataSource.setUpResultAsListOfValues( + queryName, + ScalaList.of(ScalaList.of(10, new BigDecimal("1.0001"))) + ); + + getDataAndUpdateTable(queryName, schemaMapper, table); + + var existingRows = toJava(table.pullAllRows()); + assertEquals(existingRows.size(), 1); + assertEquals(existingRows.get(0).get("Id"), "10"); + assertEquals(existingRows.get(0).get("doubleValue"), 1.0001d); + } } private static void getDataAndUpdateTable(String queryName, SchemaMapper schemaMapper, FakeInMemoryTable table) throws Exception { @@ -126,9 +172,8 @@ private static RowWithData mapToRow(SchemaMapper schemaMapper, List valu } private static List> getQueryResult(String queryName) throws Exception { - var result = OptionConverters.toJava(dataSource.getAsListOfValues(queryName)) + return OptionConverters.toJava(dataSource.getAsListOfValues(queryName)) .map(listOfLists -> toJava(listOfLists.map(ScalaCollectionConverter::toJava).toList())) .orElseThrow(() -> new Exception("Query does not exist in store. make sure it is setup")); - return result; } } diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala index 484eb50df..024766c44 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala @@ -3,6 +3,7 @@ package org.finos.vuu.util.schema import org.finos.vuu.api.{ColumnBuilder, TableDef} import org.finos.vuu.core.table.Columns import org.finos.vuu.test.FakeInMemoryTable +import org.finos.vuu.util.types.{TypeConverter, TypeConverterContainerBuilder} class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { @@ -118,7 +119,47 @@ class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { assert(existingTableRows.head.get("notionalValue") == 10.5) } - Scenario("When table has columns with different type from fields on external entity") {} + Scenario("When table has columns with different type from fields on external entity") { + + val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() + .withField("id", classOf[Int]) + .withField("decimalValue", classOf[BigDecimal]) + .withIndex("ID_INDEX", List("id")) + .build() + + val tableDef = TableDef( + name = "MyExampleTable", + keyField = "id", + columns = new ColumnBuilder() + .addString("id") + .addDouble("doubleValue") + .build() + ) + val typeConverterContainer = TypeConverterContainerBuilder() + .withConverter(TypeConverter[BigDecimal, Double](classOf[BigDecimal], classOf[Double], _.doubleValue)) + .withConverter(TypeConverter[Double, BigDecimal](classOf[Double], classOf[BigDecimal], v => BigDecimal(v.toString))) + .build() + val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + .withFieldsMap( + Map( + "id" -> "id", + "decimalValue" -> "doubleValue", + ) + ) + .withTypeConverters(typeConverterContainer) + .build() + val table = new FakeInMemoryTable("SchemaMapTest", tableDef) + givenQueryReturns(queryName, List(List(10, BigDecimal("1.0001")))) + + getDataAndUpdateTable(table, schemaMapper, queryName) + + val existingTableRows = table.pullAllRows() + assert(existingTableRows.size == 1) + assert(existingTableRows.head.get("id") == "10") + assert(existingTableRows.head.get("doubleValue") == 1.0001d) + + } + Scenario("When query result has less number of fields than table columns return useful error") {} Scenario("When column and field order does not match but no fields map defined on schema return useful error") {} Scenario("When getting data from source fails return useful error") {}