diff --git a/legend-engine-config/legend-engine-extensions-collection-execution/pom.xml b/legend-engine-config/legend-engine-extensions-collection-execution/pom.xml index bcdd0144c84..31ec757a36e 100644 --- a/legend-engine-config/legend-engine-extensions-collection-execution/pom.xml +++ b/legend-engine-config/legend-engine-extensions-collection-execution/pom.xml @@ -158,6 +158,11 @@ org.finos.legend.engine legend-engine-xt-flatdata-shared + + + org.finos.legend.engine + legend-engine-xt-arrow-runtime + org.finos.legend.engine legend-engine-xt-flatdata-driver-bloomberg diff --git a/legend-engine-config/legend-engine-extensions-collection-execution/src/test/java/org/finos/legend/engine/extensions/collection/execution/TestExtensions.java b/legend-engine-config/legend-engine-extensions-collection-execution/src/test/java/org/finos/legend/engine/extensions/collection/execution/TestExtensions.java index b49e4c13ea2..9f7046a3d1a 100644 --- a/legend-engine-config/legend-engine-extensions-collection-execution/src/test/java/org/finos/legend/engine/extensions/collection/execution/TestExtensions.java +++ b/legend-engine-config/legend-engine-extensions-collection-execution/src/test/java/org/finos/legend/engine/extensions/collection/execution/TestExtensions.java @@ -20,6 +20,7 @@ import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.api.set.MutableSet; import org.eclipse.collections.impl.factory.Sets; +import org.finos.legend.engine.external.format.arrow.ArrowRuntimeExtension; import org.finos.legend.engine.external.format.flatdata.FlatDataRuntimeExtension; import org.finos.legend.engine.external.format.flatdata.driver.spi.FlatDataDriverDescription; import org.finos.legend.engine.external.format.json.JsonSchemaRuntimeExtension; @@ -98,7 +99,8 @@ protected MutableList> expectedE return Lists.mutable.>empty() .with(FlatDataRuntimeExtension.class) .with(JsonSchemaRuntimeExtension.class) - .with(XsdRuntimeExtension.class); + .with(XsdRuntimeExtension.class) + .with(ArrowRuntimeExtension.class); } @Test diff --git a/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml b/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml index c293f78a522..75a85f0f11b 100644 --- a/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml +++ b/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml @@ -214,7 +214,12 @@ + + + org.finos.legend.engine + legend-engine-xt-arrow-pure + org.finos.legend.engine diff --git a/legend-engine-config/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java b/legend-engine-config/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java index c010caab393..5ae46944c3d 100644 --- a/legend-engine-config/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java +++ b/legend-engine-config/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java @@ -414,6 +414,7 @@ protected Iterable> getExpectedPlan .with(XMLJavaBindingPureCoreExtension.class) .with(ServicePureCoreExtension.class) .with(RelationalJavaBindingPureCoreExtension.class) + .with(ArrowPureCoreExtension.class) ; } @@ -427,7 +428,8 @@ protected Iterable>> getExp .with(org.finos.legend.engine.external.format.protobuf.ProtobufFormatExtension.class) .with(org.finos.legend.engine.query.graphQL.api.format.GraphQLFormatExtension.class) .with(org.finos.legend.engine.query.graphQL.api.format.GraphQLSDLFormatExtension.class) - .with(org.finos.legend.engine.external.format.daml.DamlFormatExtension.class); + .with(org.finos.legend.engine.external.format.daml.DamlFormatExtension.class) + ; } protected Iterable> getExpectedFlatDataDriverDescriptionExtensions() @@ -483,6 +485,7 @@ protected Iterable getExpectedCodeRepositories() .with("core_external_format_openapi") .with("core_external_format_protobuf") .with("core_external_format_xml") + .with("core_external_format_arrow") .with("core_external_query_graphql") .with("core_external_query_graphql_metamodel") .with("core_external_query_sql_metamodel") diff --git a/legend-engine-config/legend-engine-server/pom.xml b/legend-engine-config/legend-engine-server/pom.xml index e84df8454ce..91d18efaa46 100644 --- a/legend-engine-config/legend-engine-server/pom.xml +++ b/legend-engine-config/legend-engine-server/pom.xml @@ -503,6 +503,17 @@ legend-engine-xt-flatdata-runtime runtime + + org.finos.legend.engine + legend-engine-xt-arrow-runtime + runtime + + + org.finos.legend.engine + legend-engine-xt-arrow-pure + runtime + + org.finos.legend.engine legend-engine-xt-json-model diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/ExternalFormatExecutionExtension.java b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/ExternalFormatExecutionExtension.java index 2537b0a03aa..56a6e41c968 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/ExternalFormatExecutionExtension.java +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/ExternalFormatExecutionExtension.java @@ -33,6 +33,7 @@ import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNode; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.DataQualityExecutionNode; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.ExternalFormatExternalizeExecutionNode; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.ExternalFormatExternalizeTDSExecutionNode; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.ExternalFormatInternalizeExecutionNode; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.UrlStreamExecutionNode; import org.finos.legend.engine.shared.core.url.UrlFactory; @@ -71,6 +72,11 @@ else if (executionNode instanceof ExternalFormatExternalizeExecutionNode) { return executeExternalizeExecutionNode((ExternalFormatExternalizeExecutionNode) executionNode, pm, executionState); } + + else if (executionNode instanceof ExternalFormatExternalizeTDSExecutionNode) + { + return executeExternalizeTDSExecutionNode((ExternalFormatExternalizeTDSExecutionNode) executionNode, pm, executionState); + } else { return null; @@ -104,6 +110,19 @@ private Result executeExternalizeExecutionNode(ExternalFormatExternalizeExecutio return extension.executeExternalizeExecutionNode(node, result, profiles, executionState); } + private Result executeExternalizeTDSExecutionNode(ExternalFormatExternalizeTDSExecutionNode node, MutableList profiles, ExecutionState executionState) + { + ExternalFormatRuntimeExtension extension = EXTENSIONS.get(node.contentType); + if (extension == null) + { + throw new IllegalStateException("No runtime extension for contentType " + node.contentType); + } + + Result result = node.executionNodes().getAny().accept(new ExecutionNodeExecutor(profiles, executionState)); + return extension.executeExternalizeTDSExecutionNode(node, result, profiles, executionState); + } + + private Result executeUrlStream(UrlStreamExecutionNode node, MutableList profiles, ExecutionState executionState) { try diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/ExternalFormatRuntimeExtension.java b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/ExternalFormatRuntimeExtension.java index ee8b0d5f13a..5d1c07d5dd9 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/ExternalFormatRuntimeExtension.java +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/ExternalFormatRuntimeExtension.java @@ -19,6 +19,7 @@ import org.finos.legend.engine.plan.execution.result.Result; import org.finos.legend.engine.plan.execution.result.object.StreamingObjectResult; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.ExternalFormatExternalizeExecutionNode; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.ExternalFormatExternalizeTDSExecutionNode; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.ExternalFormatInternalizeExecutionNode; import org.pac4j.core.profile.CommonProfile; @@ -38,4 +39,9 @@ default Result executeExternalizeExecutionNode(ExternalFormatExternalizeExecutio { throw new UnsupportedOperationException("Externalize not supported by format - " + node.contentType); } + + default Result executeExternalizeTDSExecutionNode(ExternalFormatExternalizeTDSExecutionNode node, Result result, MutableList profiles, ExecutionState executionState) + { + throw new UnsupportedOperationException("Externalize TDS not supported by format - " + node.contentType); + } } diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/CorePureProtocolExtension.java b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/CorePureProtocolExtension.java index f6380e6962d..c2c8f984e6f 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/CorePureProtocolExtension.java +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/CorePureProtocolExtension.java @@ -29,6 +29,7 @@ import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNode; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.DataQualityExecutionNode; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.ExternalFormatExternalizeExecutionNode; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.ExternalFormatExternalizeTDSExecutionNode; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.ExternalFormatInternalizeExecutionNode; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.UrlStreamExecutionNode; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.VariableResolutionExecutionNode; @@ -130,6 +131,8 @@ public List>>> getExtraProtocolSubTypeInfo .withSubtype(VariableResolutionExecutionNode.class, "varResolution") .withSubtype(ExternalFormatInternalizeExecutionNode.class, "externalFormatInternalize") .withSubtype(ExternalFormatExternalizeExecutionNode.class, "externalFormatExternalize") + .withSubtype(ExternalFormatExternalizeTDSExecutionNode.class, "externalFormatExternalizeTDS") + .build(), ProtocolSubTypeInfo.newBuilder(TestSuite.class) .withSubtype(MappingTestSuite.class, "mappingTestSuite") diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/executionPlan/nodes/externalFormat/ExternalFormatExternalizeTDSExecutionNode.java b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/executionPlan/nodes/externalFormat/ExternalFormatExternalizeTDSExecutionNode.java new file mode 100644 index 00000000000..945e638b534 --- /dev/null +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-pure/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/executionPlan/nodes/externalFormat/ExternalFormatExternalizeTDSExecutionNode.java @@ -0,0 +1,29 @@ +// Copyright 2022 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat; + +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNode; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNodeVisitor; + +public class ExternalFormatExternalizeTDSExecutionNode extends ExecutionNode +{ + public String contentType; + + @Override + public T accept(ExecutionNodeVisitor executionNodeVisitor) + { + return executionNodeVisitor.visit(this); + } +} diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/executionPlan/executionPlan_generation.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/executionPlan/executionPlan_generation.pure index 966ae236995..3a6f83cb2e7 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/executionPlan/executionPlan_generation.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/executionPlan/executionPlan_generation.pure @@ -144,29 +144,12 @@ function <> meta::external::format::shared::executionPlan::exter { let parameters = $fe.parametersValues->evaluateAndDeactivate(); let children = $parameters->at(0)->processValueSpecification($state, $extensions, $debug)->toOneMany(); - - let inScopeVars = $state.inScopeVars; - - let tree = #{TabularDataSet{rows}}#; - - let inputType = $parameters->at(0)->byPassValueSpecificationWrapper()->cast(@SimpleFunctionExpression).genericType.rawType; - let checked = $inputType == meta::pure::dataQuality::Checked; - - assert($tree.class == $inputType, | 'Input type \'' + $inputType->toOne()->elementToPath() + '\' and root tree type \'' + $tree.class->toOne()->elementToPath() + '\' for externalize does not match'); - - let bindingArg = $fe.parametersValues->at(1)->byPassValueSpecificationWrapper(); - let binding = $bindingArg->cast(@InstanceValue).values->match([ - b:meta::external::format::shared::binding::Binding[1] | $b, - s:String[1] | ^meta::external::format::shared::binding::Binding(name = 'generatedBinding', package = meta::external::format::shared::executionPlan, contentType = $s, modelUnit = ^meta::pure::model::unit::ModelUnit(packageableElementIncludes = extractPackageableElementFromTree($tree))) - ]); - - ^ExternalFormatExternalizeExecutionNode + let contentType = $fe.parametersValues->at(1)->byPassValueSpecificationWrapper()->cast(@InstanceValue).values->at(0)->cast(@String); + ^ExternalFormatExternalizeTDSExecutionNode ( resultType = ^ResultType(type=String), resultSizeRange = PureOne, - checked = $checked, - binding = $binding, - tree = $tree, + contentType = $contentType, executionNodes = $children ); } diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/executionPlan/executionPlan_print.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/executionPlan/executionPlan_print.pure index 7325148a4a9..1c98179867e 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/executionPlan/executionPlan_print.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/executionPlan/executionPlan_print.pure @@ -33,6 +33,14 @@ function meta::external::format::shared::executionPlan::toString::printPlanNodeT $node.implementation->printImplementation('implementation', $space+' ', $extensions) + $space + ')\n' }, + {node:ExternalFormatExternalizeTDSExecutionNode[1] | + 'ExternalFormat_ExternalizeTDS\n' + + $space + '(' + header($node, $space, $extensions) + '\n' + + $space + ' contentType = ' + $node.contentType + '\n' + + $node->childrenToString($space+' ', $extensions) + '\n' + + $node.implementation->printImplementation('implementation', $space+' ', $extensions) + + $space + ')\n' + }, {node:ExternalFormatInternalizeExecutionNode[1] | 'ExternalFormat_Internalize\n' + $space + '(' + header($node, $space, $extensions) + '\n' + diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/executionPlan/model.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/executionPlan/model.pure index 54a57e7c0e7..c7bce7b879f 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/executionPlan/model.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/executionPlan/model.pure @@ -34,6 +34,12 @@ Class meta::external::format::shared::executionPlan::ExternalFormatExternalizeEx config : ExternalFormatExternalizeConfig[0..1]; } +Class meta::external::format::shared::executionPlan::ExternalFormatExternalizeTDSExecutionNode extends ExecutionNode +{ + contentType : String[1]; + config : ExternalFormatExternalizeConfig[0..1]; +} + Class meta::external::format::shared::executionPlan::ExternalFormatInternalizeExecutionNode extends ExecutionNode { binding : Binding[1]; diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/protocols/pure/vX_X_X/extension/extension.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/protocols/pure/vX_X_X/extension/extension.pure index 832280fea7d..18df907e5dd 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/protocols/pure/vX_X_X/extension/extension.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/protocols/pure/vX_X_X/extension/extension.pure @@ -24,7 +24,9 @@ function meta::protocols::pure::vX_X_X::external::shared::format::serializerExte {mapping:Mapping[1], extensions:Extension[*] | [ d:meta::external::format::shared::executionPlan::ExternalFormatExternalizeExecutionNode[1] | transformExternalFormatExternalizeExecutionNode($d, $mapping, $extensions), - s:meta::external::format::shared::executionPlan::ExternalFormatInternalizeExecutionNode[1] | transformExternalFormatInternalizeExecutionNode($s, $mapping, $extensions) + s:meta::external::format::shared::executionPlan::ExternalFormatInternalizeExecutionNode[1] | transformExternalFormatInternalizeExecutionNode($s, $mapping, $extensions), + t:meta::external::format::shared::executionPlan::ExternalFormatExternalizeTDSExecutionNode[1] | transformExternalFormatExternalizeTDSExecutionNode($t, $mapping, $extensions) + ] } ); diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/protocols/pure/vX_X_X/models/executionPlan.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/protocols/pure/vX_X_X/models/executionPlan.pure index ab78a330040..781be15be0b 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/protocols/pure/vX_X_X/models/executionPlan.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/protocols/pure/vX_X_X/models/executionPlan.pure @@ -25,6 +25,11 @@ Class meta::protocols::pure::vX_X_X::metamodel::external::shared::format::execut binding : String[1]; } +Class meta::protocols::pure::vX_X_X::metamodel::external::shared::format::executionPlan::ExternalFormatExternalizeTDSExecutionNode extends ExecutionNode +{ + contentType : String[1]; +} + Class meta::protocols::pure::vX_X_X::metamodel::external::shared::format::executionPlan::ExternalFormatInternalizeExecutionNode extends ExecutionNode { contentType : String[1]; diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/protocols/pure/vX_X_X/transfers/executionPlan.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/protocols/pure/vX_X_X/transfers/executionPlan.pure index 7fe4c70f875..006ad0a71dc 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/protocols/pure/vX_X_X/transfers/executionPlan.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/protocols/pure/vX_X_X/transfers/executionPlan.pure @@ -29,6 +29,16 @@ function meta::protocols::pure::vX_X_X::transformation::fromPureGraph::external: ); } +function meta::protocols::pure::vX_X_X::transformation::fromPureGraph::external::shared::format::transformExternalFormatExternalizeTDSExecutionNode(node:meta::external::format::shared::executionPlan::ExternalFormatExternalizeTDSExecutionNode[1], mapping:meta::pure::mapping::Mapping[1], extensions:Extension[*]): ExecutionNode[1] +{ + ^meta::protocols::pure::vX_X_X::metamodel::external::shared::format::executionPlan::ExternalFormatExternalizeTDSExecutionNode( + _type = 'externalFormatExternalizeTDS', + resultType = $node.resultType->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::executionPlan::transformResultType($mapping, $extensions), + resultSizeRange = $node.resultSizeRange->map(s| $s->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::domain::transformMultiplicity()), + contentType = $node.contentType + ); +} + function meta::protocols::pure::vX_X_X::transformation::fromPureGraph::external::shared::format::transformExternalFormatInternalizeExecutionNode(node:meta::external::format::shared::executionPlan::ExternalFormatInternalizeExecutionNode[1], mapping:meta::pure::mapping::Mapping[1], extensions:Extension[*]): ExecutionNode[1] { ^meta::protocols::pure::vX_X_X::metamodel::external::shared::format::executionPlan::ExternalFormatInternalizeExecutionNode( diff --git a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java index d6b0cf6581b..2eea4dbd2b9 100644 --- a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java +++ b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java @@ -100,7 +100,8 @@ protected MutableList buildRepositories(SourceLocationCon .with(this.buildCore("legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-javaPlatformBinding-pure","nonrelational-mongodb-java-platform-binding")) .with(this.buildCore("legend-engine-xts-service/legend-engine-language-pure-dsl-service-pure","service")) .with(this.buildCore("legend-engine-xts-iceberg/legend-engine-xt-iceberg-pure","external-tableformat-iceberg")) - ; + .with(this.buildCore("legend-engine-xts-arrow/legend-engine-xt-arrow-pure", "external-format-arrow")) + ; } @Override diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/pom.xml b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/pom.xml new file mode 100644 index 00000000000..235efa3dbbc --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/pom.xml @@ -0,0 +1,151 @@ + + + + + org.finos.legend.engine + legend-engine-xts-arrow + 4.28.2-SNAPSHOT + + 4.0.0 + + Legend Engine - XT - Arrow - PAR/JAVA + legend-engine-xt-arrow-pure + + + + + org.finos.legend.pure + legend-pure-maven-generation-par + + src/main/resources + ${legend.pure.version} + + platform + platform_functions + core + core_external_format_arrow + + + ${project.basedir}/src/main/resources/core_external_format_arrow.definition.json + + + + + generate-sources + + build-pure-jar + + + + + + org.finos.legend.pure + legend-pure-m2-dsl-diagram-grammar + ${legend.pure.version} + + + org.finos.legend.engine + legend-engine-pure-code-compiled-core + ${project.version} + + + + + org.finos.legend.pure + legend-pure-maven-generation-java + + + compile + + build-pure-compiled-jar + + + true + true + modular + true + + core_external_format_arrow + + + + + + + org.finos.legend.pure + legend-pure-m2-dsl-diagram-grammar + ${legend.pure.version} + + + org.finos.legend.engine + legend-engine-pure-code-compiled-core + ${project.version} + + + + + + + + + + org.finos.legend.pure + legend-pure-m4 + + + org.finos.legend.pure + legend-pure-m3-core + + + org.finos.legend.pure + legend-pure-runtime-java-engine-compiled + + + org.finos.legend.engine + legend-engine-pure-platform-java + + + + org.eclipse.collections + eclipse-collections + + + org.eclipse.collections + eclipse-collections-api + compile + + + org.finos.legend.engine + legend-engine-pure-code-compiled-core + + + org.finos.legend.engine + legend-engine-pure-code-core-extension + + + org.finos.legend.engine + legend-engine-language-pure-compiler + test + + + + junit + junit + + + + diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/java/org/finos/legend/pure/code/core/ArrowPureCoreExtension.java b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/java/org/finos/legend/pure/code/core/ArrowPureCoreExtension.java new file mode 100644 index 00000000000..1b6e1143a66 --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/java/org/finos/legend/pure/code/core/ArrowPureCoreExtension.java @@ -0,0 +1,32 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.code.core; + +import org.finos.legend.engine.pure.code.core.PureCoreExtension; + +public class ArrowPureCoreExtension implements PureCoreExtension +{ + @Override + public String functionFile() + { + return "core_external_format_arrow/contract.pure"; + } + + @Override + public String functionSignature() + { + return "meta::external::format::arrow::extension::arrowFormatExtension__Extension_1_"; + } +} diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/java/org/finos/legend/pure/code/core/CoreArrowCodeRepositoryProvider.java b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/java/org/finos/legend/pure/code/core/CoreArrowCodeRepositoryProvider.java new file mode 100644 index 00000000000..93cb21091aa --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/java/org/finos/legend/pure/code/core/CoreArrowCodeRepositoryProvider.java @@ -0,0 +1,28 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.code.core; + +import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepository; +import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider; +import org.finos.legend.pure.m3.serialization.filesystem.repository.GenericCodeRepository; + +public class CoreArrowCodeRepositoryProvider implements CodeRepositoryProvider +{ + @Override + public CodeRepository repository() + { + return GenericCodeRepository.build("core_external_format_arrow.definition.json"); + } +} diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/META-INF/services/org.finos.legend.engine.pure.code.core.PureCoreExtension b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/META-INF/services/org.finos.legend.engine.pure.code.core.PureCoreExtension new file mode 100644 index 00000000000..b4ccbd21abf --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/META-INF/services/org.finos.legend.engine.pure.code.core.PureCoreExtension @@ -0,0 +1 @@ +org.finos.legend.pure.code.core.ArrowPureCoreExtension \ No newline at end of file diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider new file mode 100644 index 00000000000..1bd5c15c8d3 --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider @@ -0,0 +1 @@ +org.finos.legend.pure.code.core.CoreArrowCodeRepositoryProvider \ No newline at end of file diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/core_external_format_arrow.definition.json b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/core_external_format_arrow.definition.json new file mode 100644 index 00000000000..bf38ed96cba --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/core_external_format_arrow.definition.json @@ -0,0 +1,9 @@ +{ + "name": "core_external_format_arrow", + "pattern": "(meta::external::format::arrow|meta::protocols::pure)(::.*)?", + "dependencies": [ + "platform", + "platform_functions", + "core" + ] +} diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/core_external_format_arrow/contract.pure b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/core_external_format_arrow/contract.pure new file mode 100644 index 00000000000..910c7d5dacd --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/core_external_format_arrow/contract.pure @@ -0,0 +1,39 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::extension::*; +import meta::external::format::shared::*; + + +function meta::external::format::arrow::contract::arrowFormatContract(): ExternalFormatContract[1] +{ + ^ExternalFormatContract + ( + id = 'Arrow', + contentTypes = 'application/x.arrow', + + externalFormatMetamodel = meta::external::format::arrow::metamodel::Schema + ) +} + +function meta::external::format::arrow::extension::arrowFormatExtension(): Extension[1] +{ + ^Extension + ( + type = 'External Format - Arrow', + availableExternalFormats = meta::external::format::arrow::contract::arrowFormatContract() + ) +} + + diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/core_external_format_arrow/metamodel.pure b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/core_external_format_arrow/metamodel.pure new file mode 100644 index 00000000000..657d1891ad5 --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/main/resources/core_external_format_arrow/metamodel.pure @@ -0,0 +1,38 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +Class meta::external::format::arrow::metamodel::Schema +{ + fields:meta::external::format::arrow::metamodel::Field[*]; +} + +Class meta::external::format::arrow::metamodel::Field +{ + name:String[1]; + nullable:Boolean[1]; + type:meta::external::format::arrow::metamodel::ArrowType[1]; + children:meta::external::format::arrow::metamodel::Field[*]; + +} +Class meta::external::format::arrow::metamodel::ArrowType +{ +} + +Class meta::external::format::arrow::metamodel::PrimitiveType extends meta::external::format::arrow::metamodel::ArrowType +{ +} + +Class meta::external::format::arrow::metamodel::ComplexType extends meta::external::format::arrow::metamodel::ArrowType +{ +} \ No newline at end of file diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/test/java/org/finos/legend/pure/code/core/external/format/arrowCore/TestExtensionAvailable.java b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/test/java/org/finos/legend/pure/code/core/external/format/arrowCore/TestExtensionAvailable.java new file mode 100644 index 00000000000..e41e033537d --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/src/test/java/org/finos/legend/pure/code/core/external/format/arrowCore/TestExtensionAvailable.java @@ -0,0 +1,34 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.code.core.external.format.arrowCore; + +import org.eclipse.collections.api.list.MutableList; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.pure.code.core.PureCoreExtension; +import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; +import org.finos.legend.pure.code.core.ArrowPureCoreExtension; +import org.junit.Assert; +import org.junit.Test; + +public class TestExtensionAvailable +{ + @Test + public void testServiceAvailable() + { + MutableList extensions = PureCoreExtensionLoader.extensions(); + Assert.assertEquals(1, extensions.selectInstancesOf(ArrowPureCoreExtension.class).get(0).extraPureCoreExtensions(PureModel.CORE_PURE_MODEL.getExecutionSupport()).size()); + Assert.assertEquals("External Format - Arrow", extensions.get(0).extraPureCoreExtensions(PureModel.CORE_PURE_MODEL.getExecutionSupport()).getFirst()._type()); + } +} diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/pom.xml b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/pom.xml new file mode 100644 index 00000000000..ab24d4caaba --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/pom.xml @@ -0,0 +1,140 @@ + + + + org.finos.legend.engine + legend-engine-xts-arrow + 4.28.2-SNAPSHOT + + 4.0.0 + + legend-engine-xt-arrow-runtime + jar + Legend Engine - XT - Arrow - Runtime + + + org.finos.legend.engine + legend-engine-executionPlan-execution + + + org.finos.legend.engine + legend-engine-xt-relationalStore-executionPlan + + + org.finos.legend.engine + legend-engine-external-shared-format-runtime + + + + org.finos.legend.engine + legend-engine-xt-relationalStore-protocol + + + org.finos.legend.engine + legend-engine-protocol-pure + + + + org.finos.legend.engine + legend-engine-shared-core + + + org.eclipse.collections + eclipse-collections-api + + + org.eclipse.collections + eclipse-collections + + + + org.pac4j + pac4j-core + + + + org.apache.arrow + arrow-vector + + + org.apache.arrow + arrow-memory-core + + + org.apache.arrow + arrow-jdbc + + + + + + com.h2database + h2 + ${h2.version} + runtime + + + org.mockito + mockito-core + test + + + junit + junit + test + + + org.finos.legend.engine + legend-engine-language-pure-compiler + test + + + org.finos.legend.engine + legend-engine-executionPlan-generation + test + + + org.finos.legend.engine + legend-engine-xt-arrow-pure + test + + + org.finos.legend.engine + legend-engine-xt-relationalStore-javaPlatformBinding-pure + test + + + + org.finos.legend.engine + legend-engine-pure-code-compiled-core + test + + + com.fasterxml.jackson.core + jackson-databind + test + + + org.finos.legend.pure + legend-pure-m3-core + test + + + org.finos.legend.engine + legend-engine-xt-relationalStore-grammar + test + + + + org.finos.legend.engine + legend-engine-pure-platform-dsl-mapping-java + test + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + test + + + + + \ No newline at end of file diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/finos/legend/engine/external/format/arrow/ArrowDataWriter.java b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/finos/legend/engine/external/format/arrow/ArrowDataWriter.java new file mode 100644 index 00000000000..012a4973a9e --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/finos/legend/engine/external/format/arrow/ArrowDataWriter.java @@ -0,0 +1,80 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.external.format.arrow; + +import java.util.Calendar; +import java.util.Locale; +import java.util.TimeZone; +import org.apache.arrow.adapter.jdbc.ArrowVectorIterator; +import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.JdbcToArrowConfig; +import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.finos.legend.engine.external.shared.runtime.write.ExternalFormatWriter; + +import java.io.IOException; +import java.io.OutputStream; +import java.sql.ResultSet; +import java.sql.SQLException; + +public class ArrowDataWriter extends ExternalFormatWriter implements AutoCloseable +{ + private final ArrowVectorIterator iterator; + private final BufferAllocator allocator; + + public ArrowDataWriter(ResultSet resultSet) throws SQLException, IOException + { + this.allocator = new RootAllocator(); + JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(allocator, Calendar.getInstance(TimeZone.getDefault(), Locale.ROOT)).build(); + this.iterator = JdbcToArrow.sqlToArrowVectorIterator(resultSet, config); + + } + + + @Override + public void writeData(OutputStream outputStream) throws IOException + { + + try + { + while (this.iterator.hasNext()) + { + try (VectorSchemaRoot vector = iterator.next(); + ArrowStreamWriter writer = new ArrowStreamWriter(vector, null, outputStream) + ) + { + writer.start(); + writer.writeBatch(); + } + } + } + catch (Exception e) + { + this.iterator.close(); + throw e; + } + + } + + @Override + public void close() + { + allocator.close(); + } +} + diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/finos/legend/engine/external/format/arrow/ArrowRuntimeExtension.java b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/finos/legend/engine/external/format/arrow/ArrowRuntimeExtension.java new file mode 100644 index 00000000000..3aba69cc8e5 --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/finos/legend/engine/external/format/arrow/ArrowRuntimeExtension.java @@ -0,0 +1,73 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +package org.finos.legend.engine.external.format.arrow; + +import org.eclipse.collections.api.list.MutableList; +import org.finos.legend.engine.external.shared.runtime.ExternalFormatRuntimeExtension; +import org.finos.legend.engine.external.shared.runtime.write.ExternalFormatSerializeResult; +import org.finos.legend.engine.plan.execution.nodes.state.ExecutionState; +import org.finos.legend.engine.plan.execution.result.Result; +import org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.ExternalFormatExternalizeTDSExecutionNode; +import org.pac4j.core.profile.CommonProfile; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; + + +public class ArrowRuntimeExtension implements ExternalFormatRuntimeExtension +{ + private static final String CONTENT_TYPE = "application/x.arrow"; + + @Override + public List getContentTypes() + { + return Collections.singletonList(CONTENT_TYPE); + } + + @Override + public Result executeExternalizeTDSExecutionNode(ExternalFormatExternalizeTDSExecutionNode node, Result result, MutableList profiles, ExecutionState executionState) + { + try + { + if (result instanceof RelationalResult) + { + return streamArrowFromRelational((RelationalResult) result); + } + else + { + throw new RuntimeException("Arrow external format only supported on relational execution"); + + } + + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + private Result streamArrowFromRelational(RelationalResult relationalResult) throws SQLException, IOException + { + + return new ExternalFormatSerializeResult(new ArrowDataWriter(relationalResult.getResultSet()), relationalResult); + + } + + +} \ No newline at end of file diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/resources/META-INF/services/org.finos.legend.engine.external.shared.runtime.ExternalFormatRuntimeExtension b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/resources/META-INF/services/org.finos.legend.engine.external.shared.runtime.ExternalFormatRuntimeExtension new file mode 100644 index 00000000000..5f93f680f79 --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/resources/META-INF/services/org.finos.legend.engine.external.shared.runtime.ExternalFormatRuntimeExtension @@ -0,0 +1 @@ +org.finos.legend.engine.external.format.arrow.ArrowRuntimeExtension \ No newline at end of file diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/java/TestArrowNodeExecutor.java b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/java/TestArrowNodeExecutor.java new file mode 100644 index 00000000000..dd39c8d4504 --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/java/TestArrowNodeExecutor.java @@ -0,0 +1,130 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import java.io.FileOutputStream; +import java.io.IOException; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowFileWriter; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.eclipse.collections.impl.list.mutable.FastList; +import org.finos.legend.engine.external.format.arrow.ArrowRuntimeExtension; +import org.finos.legend.engine.external.shared.runtime.write.ExternalFormatSerializeResult; +import org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat; +import org.finos.legend.engine.plan.execution.stores.relational.activity.RelationalExecutionActivity; +import org.finos.legend.engine.plan.execution.stores.relational.result.RelationalResult; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.RelationalExecutionNode; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.externalFormat.ExternalFormatExternalizeTDSExecutionNode; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.result.SQLResultColumn; +import org.finos.legend.engine.shared.core.api.request.RequestContext; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.sql.Connection; +import java.sql.DriverManager; + +import static org.mockito.ArgumentMatchers.any; + +public class TestArrowNodeExecutor + +{ + @Test + public void testExternalize() throws Exception + { + ArrowRuntimeExtension extension = new ArrowRuntimeExtension(); + ExternalFormatExternalizeTDSExecutionNode node = new ExternalFormatExternalizeTDSExecutionNode(); + //create a real result from H2 + RelationalExecutionNode mockExecutionNode = Mockito.mock(RelationalExecutionNode.class); + DatabaseConnection mockDatabaseConnection = Mockito.mock(DatabaseConnection.class); + + mockExecutionNode.connection = mockDatabaseConnection; + Mockito.when(mockDatabaseConnection.accept(any())).thenReturn(false); + try (Connection conn = DriverManager.getConnection("jdbc:h2:~/test", "sa", ""); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) + { + //setup table + conn.createStatement().execute("DROP TABLE IF EXISTS testtable"); + conn.createStatement().execute("Create Table testtable (testInt INTEGER, testString VARCHAR(255), testDate TIMESTAMP, testBool BOOLEAN)"); + conn.createStatement().execute("INSERT INTO testtable (testInt, testString, testDate, testBool) VALUES(1,'A', '2020-01-01 00:00:00-05:00',true),( 2,'B', '2020-01-01 00:00:00-02:00',false ),( 3,'B', '2020-01-01 00:00:00-05:00',false )"); + + RelationalResult result = new RelationalResult(FastList.newListWith(new RelationalExecutionActivity("SELECT * FROM testtable", null)), mockExecutionNode, FastList.newListWith(new SQLResultColumn("testInt", "INTEGER"), new SQLResultColumn("testString", "VARCHAR"), new SQLResultColumn("testDate", "TIMESTAMP"), new SQLResultColumn("testBool", "TIMESTAMP")), null, "GMT", conn, null, null, null, new RequestContext()); + + ExternalFormatSerializeResult nodeExecute = (ExternalFormatSerializeResult) extension.executeExternalizeTDSExecutionNode(node, result, null, null); + + nodeExecute.stream(outputStream, SerializationFormat.DEFAULT); + assertArrow(outputStream, "TESTINT\tTESTSTRING\tTESTDATE\tTESTBOOL\n" + + "1\tA\t1577854800000\ttrue\n" + + "2\tB\t1577844000000\tfalse\n" + + "3\tB\t1577854800000\tfalse\n"); + } + + } + + private void assertArrow(ByteArrayOutputStream actualOutputStream, String expectedTSV) throws IOException //input a TSV String + { + actualOutputStream.flush(); + ByteArrayInputStream input = new ByteArrayInputStream(actualOutputStream.toByteArray()); + actualOutputStream.close(); + + VectorSchemaRoot actualSchema = null; + String actualTSV = ""; + try ( + BufferAllocator rootAllocator = new RootAllocator(); + ArrowStreamReader actualReader = new ArrowStreamReader(input, rootAllocator) + + ) + { + while (actualReader.loadNextBatch()) + { + actualSchema = actualReader.getVectorSchemaRoot(); + actualTSV += actualSchema.contentToTSVString(); + } + } + catch (Exception e) + { + e.printStackTrace(); + } + Assert.assertEquals(expectedTSV, actualTSV); + + + } + + /* + * This is a utility for generating arrow raw data to a file for test setup + */ + protected void arrowFileCreationUtility(VectorSchemaRoot vectorSchemaRoot, String file) + { + + try ( + FileOutputStream fileOutputStream = new FileOutputStream(file); + ArrowFileWriter writer = new ArrowFileWriter(vectorSchemaRoot, null, fileOutputStream.getChannel()) + ) + { + writer.start(); + writer.writeBatch(); + writer.end(); + } + catch (IOException e) + { + e.printStackTrace(); + } + + } + +} diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/java/TestArrowQueries.java b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/java/TestArrowQueries.java new file mode 100644 index 00000000000..0b5af0ad195 --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/java/TestArrowQueries.java @@ -0,0 +1,118 @@ +/* + * // Copyright 2023 Goldman Sachs + * // + * // Licensed under the Apache License, Version 2.0 (the "License"); + * // you may not use this file except in compliance with the License. + * // You may obtain a copy of the License at + * // + * // http://www.apache.org/licenses/LICENSE-2.0 + * // + * // Unless required by applicable law or agreed to in writing, software + * // distributed under the License is distributed on an "AS IS" BASIS, + * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * // See the License for the specific language governing permissions and + * // limitations under the License. + */ + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.HelperRuntimeBuilder; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.HelperValueSpecificationBuilder; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.plan.execution.PlanExecutor; +import org.finos.legend.engine.plan.execution.result.StreamingResult; +import org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat; +import org.finos.legend.engine.plan.execution.stores.relational.plugin.RelationalStoreExecutor; +import org.finos.legend.engine.plan.execution.stores.relational.plugin.RelationalStoreExecutorBuilder; +import org.finos.legend.engine.plan.generation.PlanGenerator; +import org.finos.legend.engine.plan.generation.transformers.LegendPlanTransformers; +import org.finos.legend.engine.plan.platform.PlanPlatform; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.finos.legend.engine.shared.core.api.model.ExecuteInput; +import org.finos.legend.engine.shared.core.deployment.DeploymentMode; +import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; +import org.finos.legend.pure.generated.Root_meta_pure_runtime_Runtime; +import org.finos.legend.pure.generated.core_pure_binding_extension; +import org.finos.legend.pure.generated.core_relational_relational_extensions_extension; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; +import org.junit.Assert; +import org.junit.Test; + +import static org.finos.legend.pure.generated.core_external_format_arrow_contract.Root_meta_external_format_arrow_extension_arrowFormatExtension__Extension_1_; +import static org.finos.legend.pure.generated.core_relational_java_platform_binding_legendJavaPlatformBinding_relationalLegendJavaPlatformBindingExtension.Root_meta_relational_executionPlan_platformBinding_legendJava_relationalExtensionJavaPlatformBinding__Extension_1_; + +public class TestArrowQueries +{ + + @Test + public void runTest() + { + try ( + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ) + { + ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports(); + ExecuteInput input = objectMapper.readValue(getClass().getClassLoader().getResource("arrowService.json"), ExecuteInput.class); + + PureModel model = org.finos.legend.engine.language.pure.compiler.Compiler.compile((PureModelContextData) input.model, DeploymentMode.TEST, null); + Root_meta_pure_runtime_Runtime runtime = HelperRuntimeBuilder.buildPureRuntime(input.runtime, model.getContext()); + + LambdaFunction lambda = HelperValueSpecificationBuilder.buildLambda(input.function.body, input.function.parameters, model.getContext()); + MutableList extensions = Lists.mutable.with(core_pure_binding_extension.Root_meta_external_format_shared_externalFormatExtension__Extension_1_(model.getExecutionSupport())); + extensions.add(Root_meta_relational_executionPlan_platformBinding_legendJava_relationalExtensionJavaPlatformBinding__Extension_1_(model.getExecutionSupport())); + extensions.add(core_relational_relational_extensions_extension.Root_meta_relational_extension_relationalExtension__Extension_1_(model.getExecutionSupport())); + extensions.add(Root_meta_external_format_arrow_extension_arrowFormatExtension__Extension_1_(model.getExecutionSupport())); + SingleExecutionPlan plan = PlanGenerator.generateExecutionPlan(lambda, model.getMapping("relationalMapping::TradeAccountRelationalMapping"), runtime, null, model, "vX_X_X", PlanPlatform.JAVA, "test", extensions, LegendPlanTransformers.transformers); + RelationalStoreExecutor relationalStoreExecutor = new RelationalStoreExecutorBuilder().build(); + PlanExecutor executor = PlanExecutor.newPlanExecutor(relationalStoreExecutor); + PlanExecutor.ExecuteArgs executeArgs = PlanExecutor.ExecuteArgs.newArgs() + .withPlan(plan) + .build(); + StreamingResult streamingResult = (StreamingResult) executor.executeWithArgs(executeArgs); + streamingResult.stream(baos, SerializationFormat.DEFAULT); + assertAndValidateArrow(new ByteArrayInputStream(baos.toByteArray()), "expectedArrowServiceData.arrow"); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + private void assertAndValidateArrow(ByteArrayInputStream actualStream, String expectedArrowFilePath) throws Exception + { + try ( + BufferAllocator rootAllocator = new RootAllocator(); + ArrowStreamReader expectedArrowReader = new ArrowStreamReader(Files.newInputStream(Paths.get(this.getClass().getResource(expectedArrowFilePath).toURI())), rootAllocator); + ArrowStreamReader actualArrowReader = new ArrowStreamReader(actualStream, rootAllocator) + ) + { + StringBuilder expected = new StringBuilder(); + StringBuilder actual = new StringBuilder(); + while (expectedArrowReader.loadNextBatch()) + { + VectorSchemaRoot expectedVectorSchemaRoot = expectedArrowReader.getVectorSchemaRoot(); + expected.append(expectedVectorSchemaRoot.contentToTSVString()); + } + while (actualArrowReader.loadNextBatch()) + { + VectorSchemaRoot actualArrowReaderVectorSchemaRoot = actualArrowReader.getVectorSchemaRoot(); + actual.append(actualArrowReaderVectorSchemaRoot.contentToTSVString()); + } + Assert.assertEquals(expected.toString(), actual.toString()); + } + + } + +} + diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/resources/arrowService.json b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/resources/arrowService.json new file mode 100644 index 00000000000..89e3581953c --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/resources/arrowService.json @@ -0,0 +1,480 @@ +{ + "clientVersion": "vX_X_X", + "function": { + "_type": "lambda", + "body": [ + { + "_type": "func", + "function": "externalize", + "parameters": [ + { + "_type": "func", + "function": "project", + "parameters": [ + { + "_type": "func", + "function": "getAll", + "parameters": [ + { + "_type": "packageableElementPtr", + "fullPath": "relationalMapping::Trade" + } + ] + }, + { + "_type": "collection", + "multiplicity": { + "lowerBound": 5, + "upperBound": 5 + }, + "values": [ + { + "_type": "lambda", + "body": [ + { + "_type": "property", + "parameters": [ + { + "_type": "var", + "name": "x" + } + ], + "property": "active" + } + ], + "parameters": [ + { + "_type": "var", + "name": "x" + } + ] + }, + { + "_type": "lambda", + "body": [ + { + "_type": "property", + "parameters": [ + { + "_type": "var", + "name": "x" + } + ], + "property": "id" + } + ], + "parameters": [ + { + "_type": "var", + "name": "x" + } + ] + }, + { + "_type": "lambda", + "body": [ + { + "_type": "property", + "parameters": [ + { + "_type": "var", + "name": "x" + } + ], + "property": "quantity" + } + ], + "parameters": [ + { + "_type": "var", + "name": "x" + } + ] + }, + { + "_type": "lambda", + "body": [ + { + "_type": "property", + "parameters": [ + { + "_type": "var", + "name": "x" + } + ], + "property": "settlementDateTime" + } + ], + "parameters": [ + { + "_type": "var", + "name": "x" + } + ] + }, + { + "_type": "lambda", + "body": [ + { + "_type": "property", + "parameters": [ + { + "_type": "var", + "name": "x" + } + ], + "property": "tradeDate" + } + ], + "parameters": [ + { + "_type": "var", + "name": "x" + } + ] + } + ] + }, + { + "_type": "collection", + "multiplicity": { + "lowerBound": 5, + "upperBound": 5 + }, + "values": [ + { + "_type": "string", + "value": "Active" + }, + { + "_type": "string", + "value": "Id" + }, + { + "_type": "string", + "value": "Quantity" + }, + { + "_type": "string", + "value": "Settlement Date Time" + }, + { + "_type": "string", + "value": "Trade Date" + } + ] + } + ] + }, + { + "_type": "string", + "value": "application/x.arrow" + } + ] + } + ], + "parameters": [] + }, + "mapping": "relationalMapping::TradeAccountRelationalMapping", + "model": { + "_type": "data", + "elements": [ + { + "_type": "class", + "name": "Trade", + "package": "relationalMapping", + "properties": [ + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "id", + "type": "Integer" + }, + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "tradeDate", + "type": "StrictDate" + }, + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "quantity", + "type": "Float" + }, + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "active", + "type": "Boolean" + }, + { + "multiplicity": { + "lowerBound": 0, + "upperBound": 1 + }, + "name": "settlementDateTime", + "type": "DateTime" + } + ] + }, + { + "_type": "relational", + "filters": [], + "includedStores": [], + "joins": [], + "name": "TradeAccountDb", + "package": "relationalMapping", + "schemas": [ + { + "name": "default", + "tables": [ + { + "columns": [ + { + "name": "ID", + "nullable": false, + "type": { + "_type": "Integer" + } + }, + { + "name": "PROD_ID", + "nullable": true, + "type": { + "_type": "Integer" + } + }, + { + "name": "ACCOUNT_ID", + "nullable": true, + "type": { + "_type": "Integer" + } + }, + { + "name": "QUANTITY", + "nullable": true, + "type": { + "_type": "Float" + } + }, + { + "name": "TRADE_DATE", + "nullable": true, + "type": { + "_type": "Date" + } + }, + { + "name": "SETTLEMENT_DATETIME", + "nullable": true, + "type": { + "_type": "Timestamp" + } + }, + { + "name": "ACTIVE", + "nullable": true, + "type": { + "_type": "Bit" + } + } + ], + "name": "tradeTable", + "primaryKey": [ + "ID" + ] + } + ], + "views": [] + } + ] + }, + { + "_type": "mapping", + "classMappings": [ + { + "_type": "relational", + "class": "relationalMapping::Trade", + "distinct": false, + "mainTable": { + "_type": "Table", + "database": "relationalMapping::TradeAccountDb", + "mainTableDb": "relationalMapping::TradeAccountDb", + "schema": "default", + "table": "tradeTable" + }, + "primaryKey": [ + { + "_type": "column", + "column": "ID", + "table": { + "_type": "Table", + "database": "relationalMapping::TradeAccountDb", + "mainTableDb": "relationalMapping::TradeAccountDb", + "schema": "default", + "table": "tradeTable" + }, + "tableAlias": "tradeTable" + } + ], + "propertyMappings": [ + { + "_type": "relationalPropertyMapping", + "property": { + "class": "relationalMapping::Trade", + "property": "id" + }, + "relationalOperation": { + "_type": "column", + "column": "ID", + "table": { + "_type": "Table", + "database": "relationalMapping::TradeAccountDb", + "mainTableDb": "relationalMapping::TradeAccountDb", + "schema": "default", + "table": "tradeTable" + }, + "tableAlias": "tradeTable" + } + }, + { + "_type": "relationalPropertyMapping", + "property": { + "class": "relationalMapping::Trade", + "property": "tradeDate" + }, + "relationalOperation": { + "_type": "column", + "column": "TRADE_DATE", + "table": { + "_type": "Table", + "database": "relationalMapping::TradeAccountDb", + "mainTableDb": "relationalMapping::TradeAccountDb", + "schema": "default", + "table": "tradeTable" + }, + "tableAlias": "tradeTable" + } + }, + { + "_type": "relationalPropertyMapping", + "property": { + "class": "relationalMapping::Trade", + "property": "active" + }, + "relationalOperation": { + "_type": "column", + "column": "ACTIVE", + "table": { + "_type": "Table", + "database": "relationalMapping::TradeAccountDb", + "mainTableDb": "relationalMapping::TradeAccountDb", + "schema": "default", + "table": "tradeTable" + }, + "tableAlias": "tradeTable" + } + }, + { + "_type": "relationalPropertyMapping", + "property": { + "class": "relationalMapping::Trade", + "property": "quantity" + }, + "relationalOperation": { + "_type": "column", + "column": "QUANTITY", + "table": { + "_type": "Table", + "database": "relationalMapping::TradeAccountDb", + "mainTableDb": "relationalMapping::TradeAccountDb", + "schema": "default", + "table": "tradeTable" + }, + "tableAlias": "tradeTable" + } + }, + { + "_type": "relationalPropertyMapping", + "property": { + "class": "relationalMapping::Trade", + "property": "settlementDateTime" + }, + "relationalOperation": { + "_type": "column", + "column": "SETTLEMENT_DATETIME", + "table": { + "_type": "Table", + "database": "relationalMapping::TradeAccountDb", + "mainTableDb": "relationalMapping::TradeAccountDb", + "schema": "default", + "table": "tradeTable" + }, + "tableAlias": "tradeTable" + } + } + ], + "root": true + } + ], + "enumerationMappings": [], + "includedMappings": [], + "name": "TradeAccountRelationalMapping", + "package": "relationalMapping", + "tests": [] + } + ] + }, + "runtime": { + "_type": "engineRuntime", + "connections": [ + { + "store": { + "path": "relationalMapping::TradeAccountDb", + "type": "STORE" + }, + "storeConnections": [ + { + "connection": { + "_type": "RelationalDatabaseConnection", + "authenticationStrategy": { + "_type": "h2Default" + }, + "databaseType": "H2", + "datasourceSpecification": { + "_type": "h2Local", + "testDataSetupSqls": [ + "\n Drop table if exists tradeTable;\n Create Table tradeTable(id INT, prod_id INT, account_id INT, quantity FLOAT, trade_date DATE, settlement_datetime TIMESTAMP, active BIT);\n insert into tradeTable(id, prod_id, account_id, quantity, trade_date, settlement_datetime) values (1, 1, 1, 100, '2021-01-01', '2021-01-01 12:00:00-05:00');\n insert into tradeTable(id, prod_id, account_id, quantity, trade_date, settlement_datetime) values (1, 1, 1, 100, '2021-01-01', '2021-01-01 12:00:00+02:00')\n\n" + ] + }, + "element": "relationalMapping::TradeAccountDb", + "type": "H2" + }, + "id": "connection_1" + } + ] + } + ], + "mappings": [ + { + "path": "relationalMapping::TradeAccountRelationalMapping", + "type": "MAPPING" + } + ] + }, + "context": { + "_type": "BaseExecutionContext", + "queryTimeOutInSeconds": null, + "enableConstraints": true + } +} diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/resources/expectedArrowServiceData.arrow b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/resources/expectedArrowServiceData.arrow new file mode 100644 index 00000000000..e556d7a94a0 Binary files /dev/null and b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/resources/expectedArrowServiceData.arrow differ diff --git a/legend-engine-xts-arrow/pom.xml b/legend-engine-xts-arrow/pom.xml new file mode 100644 index 00000000000..b7cfc00459c --- /dev/null +++ b/legend-engine-xts-arrow/pom.xml @@ -0,0 +1,22 @@ + + + 4.0.0 + + org.finos.legend.engine + legend-engine + 4.28.2-SNAPSHOT + + + legend-engine-xts-arrow + pom + Legend Engine - XTS - Arrow + + + legend-engine-xt-arrow-pure + legend-engine-xt-arrow-runtime + + + + \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/executionPlan/tests/executionPlanTest.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/executionPlan/tests/executionPlanTest.pure index ef93295d3ab..2348187db3d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/executionPlan/tests/executionPlanTest.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/executionPlan/tests/executionPlanTest.pure @@ -2181,12 +2181,11 @@ function <> meta::pure::executionPlan::tests::datetime::testPlanWithL let extensions =meta::relational::extension::relationalExtensions() ->concatenate(meta::external::format::shared::transformation::tests::exampleExternalFormatExtension()->concatenate(meta::pure::extension::configuration::coreExtensions())); let result = executionPlan({|Product.all()->project(p|$p.name, 'name')->meta::external::format::shared::functions::externalize('text/example')}, simpleRelationalMapping,meta::relational::tests::testRuntime(), $extensions); assertEquals ( - 'ExternalFormat_Externalize\n' + + 'ExternalFormat_ExternalizeTDS\n' + '(\n' + ' type = String\n' + ' resultSizeRange = 1\n' + - ' checked = false\n' + - ' binding = meta::external::format::shared::executionPlan::generatedBinding\n' + + ' contentType = text/example\n' + '\n' + ' (\n' + ' Relational\n' + diff --git a/pom.xml b/pom.xml index 644f04a7b44..8239ea4ca17 100644 --- a/pom.xml +++ b/pom.xml @@ -54,6 +54,7 @@ legend-engine-xts-flatdata legend-engine-xts-json legend-engine-xts-avro + legend-engine-xts-arrow legend-engine-xts-iceberg @@ -198,11 +199,12 @@ 2.4.7 1.11.20 2.27.2 - 4.1.95.Final + 4.1.96.Final 2.17.129 4.6 1.2.3 0.1.5 + 13.0.0 3.8.0 @@ -2003,7 +2005,17 @@ legend-engine-xt-openapi-generation ${project.version} - + + org.finos.legend.engine + legend-engine-xt-arrow-pure + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-arrow-runtime + ${project.version} + + @@ -3017,6 +3029,35 @@ + + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + + + org.apache.arrow + arrow-memory-core + ${arrow.version} + + + org.apache.arrow + arrow-jdbc + ${arrow.version} + + +