From 1413c846a8f355acf5538b1a23ab14fd20c63b00 Mon Sep 17 00:00:00 2001 From: do4gr Date: Fri, 22 Dec 2017 13:13:35 +0100 Subject: [PATCH 1/3] reformat the date time to iso8601 in NDF --- .../client/ImportExport/BulkExport.scala | 24 ++++++++++++++++--- .../client/ImportExport/BulkImport.scala | 23 +++++++++++++++--- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkExport.scala b/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkExport.scala index 838d6acd59..0c1bf966e6 100644 --- a/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkExport.scala +++ b/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkExport.scala @@ -1,16 +1,22 @@ package cool.graph.client.ImportExport +import java.sql.Timestamp + import cool.graph.DataItem import cool.graph.Types.UserData import cool.graph.client.ClientInjector import cool.graph.client.database.{DataResolver, QueryArguments} -import cool.graph.shared.models.Project +import cool.graph.shared.models.{Project, TypeIdentifier} import spray.json.JsValue import spray.json._ + import scala.concurrent.ExecutionContext.Implicits.global import MyJsonProtocol._ + import scala.concurrent.Future import cool.graph.shared.schema.CustomScalarTypes.parseValueFromString +import org.joda.time.{DateTime, DateTimeZone} +import org.joda.time.format.DateTimeFormat class BulkExport(implicit clientInjector: ClientInjector) { @@ -101,7 +107,7 @@ class BulkExport(implicit clientInjector: ClientInjector) { val bundles = info match { case info: NodeInfo => dataItems.map(item => dataItemToExportNode(item, info)) case info: RelationInfo => dataItems.map(item => dataItemToExportRelation(item, info)) - case _: ListInfo => sys.error("shouldnt happen") + case _: ListInfo => sys.error("shouldn't happen") } val combinedElements = in.jsonElements ++ bundles.flatMap(_.jsonElements).toVector val combinedSize = bundles.map(_.size).fold(in.size) { (a, b) => @@ -141,12 +147,24 @@ class BulkExport(implicit clientInjector: ClientInjector) { val withoutHiddenFields: Map[String, Option[Any]] = dataValueMap.collect { case (k, v) if k != "createdAt" && k != "updatedAt" => (k, v) } val nonListFieldsWithValues: Map[String, Any] = withoutHiddenFields.collect { case (k, Some(v)) if !info.current.getFieldByName_!(k).isList => (k, v) } val outputMap: Map[String, Any] = nonListFieldsWithValues ++ createdAtUpdatedAtMap - val result: Map[String, Any] = Map("_typeName" -> info.current.name, "id" -> item.id) ++ outputMap + + val mapWithCorrectDateTimeFormat = outputMap.map { + case (k, v) if k == "createdAt" || k == "updatedAt" => (k, dateTimeToISO8601(v)) + case (k, v) if info.current.getFieldByName_!(k).typeIdentifier == TypeIdentifier.DateTime => (k, dateTimeToISO8601(v)) + case (k, v) => (k, v) + } + + val result: Map[String, Any] = Map("_typeName" -> info.current.name, "id" -> item.id) ++ mapWithCorrectDateTimeFormat val json = result.toJson JsonBundle(jsonElements = Vector(json), size = json.toString.length) } + private def dateTimeToISO8601(v: Any) = v.isInstanceOf[Timestamp] match { + case true => DateTime.parse(v.asInstanceOf[Timestamp].toString, DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS").withZoneUTC()) + case false => new DateTime(v.asInstanceOf[String], DateTimeZone.UTC) + } + private def dataItemToExportList(in: JsonBundle, item: DataItem, info: ListInfo): ResultFormat = { val listFieldsWithValues: Map[String, Any] = item.userData.collect { case (k, Some(v)) if info.listFields.map(p => p._1).contains(k) => (k, v) } diff --git a/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkImport.scala b/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkImport.scala index 3d919006e0..e56c3a5352 100644 --- a/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkImport.scala +++ b/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkImport.scala @@ -1,19 +1,23 @@ package cool.graph.client.ImportExport +import java.sql.Timestamp + import cool.graph.client.ClientInjector import cool.graph.client.database.DatabaseMutationBuilder.MirrorFieldDbValues import cool.graph.client.database.{DatabaseMutationBuilder, ProjectRelayId, ProjectRelayIdTable} import cool.graph.cuid.Cuid import cool.graph.shared.RelationFieldMirrorColumn import cool.graph.shared.database.Databases -import cool.graph.shared.models.{Model, Project, Relation, RelationSide} +import cool.graph.shared.models._ import slick.dbio.{DBIOAction, Effect, NoStream} import slick.jdbc.MySQLProfile.api._ import slick.lifted.TableQuery import spray.json._ import MyJsonProtocol._ -import scala.concurrent.ExecutionContext.Implicits.global +import org.joda.time.{DateTime, DateTimeZone} +import org.joda.time.format.{DateTimeFormat, ISODateTimeFormat} +import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future import scala.util.Try @@ -67,12 +71,25 @@ class BulkImport(implicit injector: ClientInjector) { ImportRelation(left, right) } + private def dateTimeFromISO8601(v: Any) = { + val string = v.asInstanceOf[String] + //"2017-12-05T12:34:23.000Z" to "2017-12-05 12:34:23.000 " which MySQL will accept + string.replace("T", " ").replace("Z", " ") + } + private def generateImportNodesDBActions(project: Project, nodes: Vector[ImportNode]): DBIOAction[Vector[Try[Int]], NoStream, Effect.Write] = { val items = nodes.map { element => val id = element.identifier.id val model = project.getModelByName_!(element.identifier.typeName) val listFields: Map[String, String] = model.scalarListFields.map(field => field.name -> "[]").toMap - val values: Map[String, Any] = element.values ++ listFields + ("id" -> id) + + val formatedDateTimes = element.values.map { + case (k, v) if k == "createdAt" || k == "updatedAt" => (k, dateTimeFromISO8601(v)) + case (k, v) if model.getFieldByName_!(k).typeIdentifier == TypeIdentifier.DateTime => (k, dateTimeFromISO8601(v)) + case (k, v) => (k, v) + } + + val values: Map[String, Any] = formatedDateTimes ++ listFields + ("id" -> id) DatabaseMutationBuilder.createDataItem(project.id, model.name, values).asTry } From 64908be36908cdac0799bbfa9ce7efde16e9927d Mon Sep 17 00:00:00 2001 From: do4gr Date: Fri, 22 Dec 2017 13:37:40 +0100 Subject: [PATCH 2/3] let surplus fields fail at db level --- .../main/scala/cool/graph/client/ImportExport/BulkImport.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkImport.scala b/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkImport.scala index e56c3a5352..dcd0fd7497 100644 --- a/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkImport.scala +++ b/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkImport.scala @@ -85,6 +85,7 @@ class BulkImport(implicit injector: ClientInjector) { val formatedDateTimes = element.values.map { case (k, v) if k == "createdAt" || k == "updatedAt" => (k, dateTimeFromISO8601(v)) + case (k, v) if !model.fields.map(_.name).contains(k) => (k, v) // let it fail at db level case (k, v) if model.getFieldByName_!(k).typeIdentifier == TypeIdentifier.DateTime => (k, dateTimeFromISO8601(v)) case (k, v) => (k, v) } From 1626bac7f7343a93d702b521119bc21d24e2a5f3 Mon Sep 17 00:00:00 2001 From: do4gr Date: Fri, 22 Dec 2017 13:44:43 +0100 Subject: [PATCH 3/3] cleanup --- .../scala/cool/graph/client/ImportExport/BulkImport.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkImport.scala b/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkImport.scala index dcd0fd7497..1ccf01629c 100644 --- a/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkImport.scala +++ b/server/client-shared/src/main/scala/cool/graph/client/ImportExport/BulkImport.scala @@ -1,8 +1,7 @@ package cool.graph.client.ImportExport -import java.sql.Timestamp - import cool.graph.client.ClientInjector +import cool.graph.client.ImportExport.MyJsonProtocol._ import cool.graph.client.database.DatabaseMutationBuilder.MirrorFieldDbValues import cool.graph.client.database.{DatabaseMutationBuilder, ProjectRelayId, ProjectRelayIdTable} import cool.graph.cuid.Cuid @@ -13,9 +12,6 @@ import slick.dbio.{DBIOAction, Effect, NoStream} import slick.jdbc.MySQLProfile.api._ import slick.lifted.TableQuery import spray.json._ -import MyJsonProtocol._ -import org.joda.time.{DateTime, DateTimeZone} -import org.joda.time.format.{DateTimeFormat, ISODateTimeFormat} import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future