diff --git a/cdap-app-fabric/pom.xml b/cdap-app-fabric/pom.xml index 3a6c817ff9cb..df6063e10bcc 100644 --- a/cdap-app-fabric/pom.xml +++ b/cdap-app-fabric/pom.xml @@ -268,7 +268,6 @@ org.hamcrest hamcrest-core - 2.2 diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/dataset/partitioned/DynamicPartitioningOutputCommitter.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/dataset/partitioned/DynamicPartitioningOutputCommitter.java index 23e0ba810e07..36004b4bf3f1 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/dataset/partitioned/DynamicPartitioningOutputCommitter.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/dataset/partitioned/DynamicPartitioningOutputCommitter.java @@ -41,6 +41,7 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.mapred.FileAlreadyExistsException; +import org.apache.hadoop.mapred.InvalidJobConfException; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -74,6 +75,15 @@ public class DynamicPartitioningOutputCommitter extends FileOutputCommitter { public DynamicPartitioningOutputCommitter(Path outputPath, TaskAttemptContext context) throws IOException { super(outputPath, context); + + //This output committer only works with `mapreduce.fileoutputcommitter.algorithm.version` = 1 + //Since hadoop 3, by default it's 2. Fail early if it's set to 2. + if (isCommitJobRepeatable(context)){ + throw new IllegalArgumentException("DynamicPartitioningOutputCommitter requires the Hadoop conf " + + "`mapreduce.fileoutputcommitter.algorithm.version` to be set to 1." + + "But Found 2."); + } + this.taskContext = context; this.jobSpecificOutputPath = outputPath; } diff --git a/cdap-formats/pom.xml b/cdap-formats/pom.xml index 5371deb0c11f..eaa4fc72ceda 100644 --- a/cdap-formats/pom.xml +++ b/cdap-formats/pom.xml @@ -55,7 +55,6 @@ com.fasterxml.jackson.core jackson-databind - 2.13.4.2 provided diff --git a/cdap-runtime-ext-dataproc/pom.xml b/cdap-runtime-ext-dataproc/pom.xml index aa066c5f24b5..ec504a3a1cb7 100644 --- a/cdap-runtime-ext-dataproc/pom.xml +++ b/cdap-runtime-ext-dataproc/pom.xml @@ -135,16 +135,6 @@ org.powermock powermock-api-mockito2 - - com.fasterxml.jackson.core - jackson-databind - compile - - - com.fasterxml.jackson.core - jackson-core - compile - diff --git a/cdap-security/pom.xml b/cdap-security/pom.xml index 56d7a6506216..2c209f6cad5b 100644 --- a/cdap-security/pom.xml +++ b/cdap-security/pom.xml @@ -89,22 +89,18 @@ org.eclipse.jetty jetty-server - ${jetty9.version} org.eclipse.jetty jetty-security - ${jetty9.version} org.eclipse.jetty jetty-util - ${jetty9.version} org.eclipse.jetty jetty-jaspi - ${jetty9.version} org.eclipse.jetty @@ -114,7 +110,6 @@ javax.servlet javax.servlet-api - 3.1.0 org.apache.geronimo.components @@ -182,12 +177,6 @@ - - commons-logging - commons-logging - 1.1.3 - test - diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/impersonation/UGIProviderTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/impersonation/UGIProviderTest.java index 6c2d0b4384ff..9f821d549948 100644 --- a/cdap-security/src/test/java/io/cdap/cdap/security/impersonation/UGIProviderTest.java +++ b/cdap-security/src/test/java/io/cdap/cdap/security/impersonation/UGIProviderTest.java @@ -124,6 +124,8 @@ public static void init() throws Exception { hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEMP_FOLDER.newFolder().getAbsolutePath()); hConf.setBoolean("ipc.client.fallback-to-simple-auth-allowed", true); hConf.setBoolean("ignore.secure.ports.for.testing", true); + hConf.setInt("dfs.namenode.metrics.logger.period.seconds", -1); + hConf.setInt("dfs.datanode.metrics.logger.period.seconds", -1); miniDFSCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build(); miniDFSCluster.waitClusterUp(); diff --git a/pom.xml b/pom.xml index 771b45e07dbb..7a6be191a069 100644 --- a/pom.xml +++ b/pom.xml @@ -142,6 +142,7 @@ 9.4.51.v20230217 2.12 4.11 + 2.2 2.20.0 1.0.5 0.10.2.2 @@ -158,7 +159,7 @@ 2.0 2.12.15 - 3.0.1 + 3.1.0 1.7.15 1.1.1.7 spark3_2.12 @@ -1117,6 +1118,12 @@ ${junit.version} test + + org.hamcrest + hamcrest-core + ${hamcrest.version} + test + pl.pragmatists JUnitParams