diff --git a/.gitignore b/.gitignore index e41a293f8..9543fc796 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ */target +target/ bin/ .project .settings/ diff --git a/pom.xml b/pom.xml index 111ccb59b..92ba40f2b 100644 --- a/pom.xml +++ b/pom.xml @@ -26,6 +26,7 @@ commons-hpcc wsclient dfsclient + spark-hpcc @@ -33,6 +34,7 @@ 3.8.0 8 3.1.1 + 3.1.2 1.6.8 false 1.6 @@ -65,6 +67,9 @@ 1.38.0 1.25.0-alpha 2.6.0 + 2.4.6 + 2.11 + 2.11 diff --git a/spark-hpcc/DataAccess/README.md b/spark-hpcc/DataAccess/README.md deleted file mode 100644 index f61198279..000000000 --- a/spark-hpcc/DataAccess/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Spark-HPCC/DataAccess -Spark-based classes for HPCC Systems/ Spark interoperability - -HPCC Systems platform target runtime must be 7.x or newer - -HPCC4J wsclient, dfsclient, and commons-hpcc build dependencies must be 7.x or newer. diff --git a/spark-hpcc/DataAccess/pom.xml b/spark-hpcc/pom.xml similarity index 75% rename from spark-hpcc/DataAccess/pom.xml rename to spark-hpcc/pom.xml index 6ff084679..1bdf4034c 100644 --- a/spark-hpcc/DataAccess/pom.xml +++ b/spark-hpcc/pom.xml @@ -2,67 +2,21 @@ 4.0.0 org.hpccsystems spark-hpcc - 9.9.0-0-SNAPSHOT + jar spark-hpcc Spark connector for reading files residing in an HPCC cluster environment - https://hpccsystems.com - - - Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - repo - - + + + org.hpccsystems + hpcc4j + 9.9.0-0-SNAPSHOT + + - 1.6.8 - false - 3.1.2 - 3.1.2 - 1.6 - true - 2.2.1 - 3.1.0 - 3.8.0 - 8 - 2.4.6 - 2.11 - 2.11 - UTF-8 net.razorvine.*:org.apache.*:org.hpccsystems.commons.*:org.hpccsystems.generated.*:org.hpccsystems.dfs.*:org.hpccsystems.ws.*:org.hpccsystems.ws.client.antlr.* - - scm:git:https://github.com/hpcc-systems/Spark-HPCC.git - scm:git:https://github.com/hpcc-systems/Spark-HPCC.git - scm:git:https://github.com/hpcc-systems/Spark-HPCC.git - HEAD - - - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - - - - ossrh - Ossrh Snapshot Repository - https://oss.sonatype.org/content/repositories/snapshots - default - - false - never - - - true - always - warn - - - @@ -155,66 +109,8 @@ provided + - - - - maven-compiler-plugin - ${maven.compiler.version} - - ${maven.compiler.release} - - - - org.apache.maven.plugins - maven-gpg-plugin - ${maven.gpg.version} - - - sign-artifacts - verify - - sign - - - - - - org.apache.maven.plugins - maven-source-plugin - ${maven.source.version} - - - attach-sources - - jar-no-fork - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven.javadoc.version} - - - attach-javadocs - - jar - - - - - none - ${javadoc.excludePackageNames} - -Xdoclint:none -
Copyright © 2021 HPCC Systems®. All rights reserved
-
Copyright © 2021 HPCC Systems®. All rights reserved
- true -
-
-
-
maven-compiler-plugin diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/FileFilterConverter.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/FileFilterConverter.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/FileFilterConverter.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/FileFilterConverter.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/GenericRowRecordAccessor.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/GenericRowRecordAccessor.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/GenericRowRecordAccessor.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/GenericRowRecordAccessor.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/GenericRowRecordBuilder.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/GenericRowRecordBuilder.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/GenericRowRecordBuilder.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/GenericRowRecordBuilder.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/HpccFile.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/HpccFile.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/HpccFile.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/HpccFile.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/HpccFileWriter.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/HpccFileWriter.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/HpccFileWriter.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/HpccFileWriter.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/HpccRDD.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/HpccRDD.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/HpccRDD.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/HpccRDD.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/PySparkField.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/PySparkField.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/PySparkField.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/PySparkField.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/PySparkFieldConstructor.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/PySparkFieldConstructor.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/PySparkFieldConstructor.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/PySparkFieldConstructor.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/RowConstructor.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/RowConstructor.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/RowConstructor.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/RowConstructor.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/SparkSchemaTranslator.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/SparkSchemaTranslator.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/SparkSchemaTranslator.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/SparkSchemaTranslator.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/Utils.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/Utils.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/Utils.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/Utils.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/datasource/HpccOptions.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/datasource/HpccOptions.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/datasource/HpccOptions.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/datasource/HpccOptions.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/datasource/HpccRelation.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/datasource/HpccRelation.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/datasource/HpccRelation.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/datasource/HpccRelation.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/datasource/HpccRelationProvider.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/datasource/HpccRelationProvider.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/datasource/HpccRelationProvider.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/datasource/HpccRelationProvider.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/datasource/package-info.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/datasource/package-info.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/datasource/package-info.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/datasource/package-info.java diff --git a/spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/package-info.java b/spark-hpcc/src/main/java/org/hpccsystems/spark/package-info.java similarity index 100% rename from spark-hpcc/DataAccess/src/main/java/org/hpccsystems/spark/package-info.java rename to spark-hpcc/src/main/java/org/hpccsystems/spark/package-info.java diff --git a/spark-hpcc/DataAccess/src/main/javadoc/overview.html b/spark-hpcc/src/main/javadoc/overview.html similarity index 100% rename from spark-hpcc/DataAccess/src/main/javadoc/overview.html rename to spark-hpcc/src/main/javadoc/overview.html diff --git a/spark-hpcc/DataAccess/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-hpcc/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister similarity index 100% rename from spark-hpcc/DataAccess/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister rename to spark-hpcc/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister diff --git a/spark-hpcc/DataAccess/src/main/resources/log4j.properties b/spark-hpcc/src/main/resources/log4j.properties similarity index 100% rename from spark-hpcc/DataAccess/src/main/resources/log4j.properties rename to spark-hpcc/src/main/resources/log4j.properties diff --git a/spark-hpcc/DataAccess/src/test/java/org/hpccsystems/spark/BaseIntegrationTest.java b/spark-hpcc/src/test/java/org/hpccsystems/spark/BaseIntegrationTest.java similarity index 100% rename from spark-hpcc/DataAccess/src/test/java/org/hpccsystems/spark/BaseIntegrationTest.java rename to spark-hpcc/src/test/java/org/hpccsystems/spark/BaseIntegrationTest.java diff --git a/spark-hpcc/DataAccess/src/test/java/org/hpccsystems/spark/DataframeIntegrationTest.java b/spark-hpcc/src/test/java/org/hpccsystems/spark/DataframeIntegrationTest.java similarity index 100% rename from spark-hpcc/DataAccess/src/test/java/org/hpccsystems/spark/DataframeIntegrationTest.java rename to spark-hpcc/src/test/java/org/hpccsystems/spark/DataframeIntegrationTest.java diff --git a/spark-hpcc/DataAccess/src/test/java/org/hpccsystems/spark/FileFilterTests.java b/spark-hpcc/src/test/java/org/hpccsystems/spark/FileFilterTests.java similarity index 100% rename from spark-hpcc/DataAccess/src/test/java/org/hpccsystems/spark/FileFilterTests.java rename to spark-hpcc/src/test/java/org/hpccsystems/spark/FileFilterTests.java diff --git a/spark-hpcc/DataAccess/src/test/java/org/hpccsystems/spark/HpccRelationIntegrationTest.java b/spark-hpcc/src/test/java/org/hpccsystems/spark/HpccRelationIntegrationTest.java similarity index 100% rename from spark-hpcc/DataAccess/src/test/java/org/hpccsystems/spark/HpccRelationIntegrationTest.java rename to spark-hpcc/src/test/java/org/hpccsystems/spark/HpccRelationIntegrationTest.java