diff --git a/cdap-app-fabric/pom.xml b/cdap-app-fabric/pom.xml
index 3a6c817ff9cb..df6063e10bcc 100644
--- a/cdap-app-fabric/pom.xml
+++ b/cdap-app-fabric/pom.xml
@@ -268,7 +268,6 @@
org.hamcrest
hamcrest-core
- 2.2
diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/dataset/partitioned/DynamicPartitioningOutputCommitter.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/dataset/partitioned/DynamicPartitioningOutputCommitter.java
index 23e0ba810e07..36004b4bf3f1 100644
--- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/dataset/partitioned/DynamicPartitioningOutputCommitter.java
+++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/dataset/partitioned/DynamicPartitioningOutputCommitter.java
@@ -41,6 +41,7 @@
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.mapred.FileAlreadyExistsException;
+import org.apache.hadoop.mapred.InvalidJobConfException;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -74,6 +75,15 @@ public class DynamicPartitioningOutputCommitter extends FileOutputCommitter {
public DynamicPartitioningOutputCommitter(Path outputPath, TaskAttemptContext context)
throws IOException {
super(outputPath, context);
+
+ //This output committer only works with `mapreduce.fileoutputcommitter.algorithm.version` = 1
+ //Since hadoop 3, by default it's 2. Fail early if it's set to 2.
+ if (isCommitJobRepeatable(context)){
+ throw new IllegalArgumentException("DynamicPartitioningOutputCommitter requires the Hadoop conf " +
+ "`mapreduce.fileoutputcommitter.algorithm.version` to be set to 1." +
+ "But Found 2.");
+ }
+
this.taskContext = context;
this.jobSpecificOutputPath = outputPath;
}
diff --git a/cdap-formats/pom.xml b/cdap-formats/pom.xml
index 5371deb0c11f..eaa4fc72ceda 100644
--- a/cdap-formats/pom.xml
+++ b/cdap-formats/pom.xml
@@ -55,7 +55,6 @@
com.fasterxml.jackson.core
jackson-databind
- 2.13.4.2
provided
diff --git a/cdap-runtime-ext-dataproc/pom.xml b/cdap-runtime-ext-dataproc/pom.xml
index aa066c5f24b5..ec504a3a1cb7 100644
--- a/cdap-runtime-ext-dataproc/pom.xml
+++ b/cdap-runtime-ext-dataproc/pom.xml
@@ -135,16 +135,6 @@
org.powermock
powermock-api-mockito2
-
- com.fasterxml.jackson.core
- jackson-databind
- compile
-
-
- com.fasterxml.jackson.core
- jackson-core
- compile
-
diff --git a/cdap-security/pom.xml b/cdap-security/pom.xml
index 56d7a6506216..2c209f6cad5b 100644
--- a/cdap-security/pom.xml
+++ b/cdap-security/pom.xml
@@ -89,22 +89,18 @@
org.eclipse.jetty
jetty-server
- ${jetty9.version}
org.eclipse.jetty
jetty-security
- ${jetty9.version}
org.eclipse.jetty
jetty-util
- ${jetty9.version}
org.eclipse.jetty
jetty-jaspi
- ${jetty9.version}
org.eclipse.jetty
@@ -114,7 +110,6 @@
javax.servlet
javax.servlet-api
- 3.1.0
org.apache.geronimo.components
@@ -182,12 +177,6 @@
-
- commons-logging
- commons-logging
- 1.1.3
- test
-
diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/impersonation/UGIProviderTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/impersonation/UGIProviderTest.java
index 6c2d0b4384ff..9f821d549948 100644
--- a/cdap-security/src/test/java/io/cdap/cdap/security/impersonation/UGIProviderTest.java
+++ b/cdap-security/src/test/java/io/cdap/cdap/security/impersonation/UGIProviderTest.java
@@ -124,6 +124,8 @@ public static void init() throws Exception {
hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEMP_FOLDER.newFolder().getAbsolutePath());
hConf.setBoolean("ipc.client.fallback-to-simple-auth-allowed", true);
hConf.setBoolean("ignore.secure.ports.for.testing", true);
+ hConf.setInt("dfs.namenode.metrics.logger.period.seconds", -1);
+ hConf.setInt("dfs.datanode.metrics.logger.period.seconds", -1);
miniDFSCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
miniDFSCluster.waitClusterUp();
diff --git a/pom.xml b/pom.xml
index 771b45e07dbb..7a6be191a069 100644
--- a/pom.xml
+++ b/pom.xml
@@ -142,6 +142,7 @@
9.4.51.v20230217
2.12
4.11
+ 2.2
2.20.0
1.0.5
0.10.2.2
@@ -158,7 +159,7 @@
2.0
2.12.15
- 3.0.1
+ 3.1.0
1.7.15
1.1.1.7
spark3_2.12
@@ -1117,6 +1118,12 @@
${junit.version}
test
+
+ org.hamcrest
+ hamcrest-core
+ ${hamcrest.version}
+ test
+
pl.pragmatists
JUnitParams