[CDAP-21047] removing log4j classes from application jar due to conflict with dataproc 2.2

[itsankit-google]

Context

2024-07-23 07:06:06,498 - INFO  [Driver:o.s.j.s.h.ContextHandler@921] - Started o.s.j.s.ServletContextHandler@1a924ce1{/,null,AVAILABLE,@Spark}
2024-07-23 07:06:07,244 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - ERROR StatusLogger Unable to create Lookup for ctx
2024-07-23 07:06:07,245 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] -  java.lang.NoSuchMethodError: 'java.lang.Object org.apache.logging.log4j.util.LoaderUtil.newCheckedInstanceOfProperty(java.lang.String, java.lang.Class, java.util.function.Supplier)'
2024-07-23 07:06:07,246 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.impl.ContextDataInjectorFactory.createInjector(ContextDataInjectorFactory.java:71)
2024-07-23 07:06:07,246 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.lookup.ContextMapLookup.<init>(ContextMapLookup.java:34)
2024-07-23 07:06:07,247 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
2024-07-23 07:06:07,247 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
2024-07-23 07:06:07,248 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
2024-07-23 07:06:07,248 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490)
2024-07-23 07:06:07,248 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.util.ReflectionUtil.instantiate(ReflectionUtil.java:188)
2024-07-23 07:06:07,249 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.lookup.Interpolator.<init>(Interpolator.java:89)
2024-07-23 07:06:07,249 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.lookup.Interpolator.<init>(Interpolator.java:108)
2024-07-23 07:06:07,250 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.config.AbstractConfiguration.<init>(AbstractConfiguration.java:136)
2024-07-23 07:06:07,250 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.config.NullConfiguration.<init>(NullConfiguration.java:32)
2024-07-23 07:06:07,250 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.LoggerContext.<clinit>(LoggerContext.java:76)
2024-07-23 07:06:07,251 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.selector.ClassLoaderContextSelector.createContext(ClassLoaderContextSelector.java:263)
2024-07-23 07:06:07,251 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.selector.ClassLoaderContextSelector.locateContext(ClassLoaderContextSelector.java:222)
2024-07-23 07:06:07,252 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.selector.ClassLoaderContextSelector.getContext(ClassLoaderContextSelector.java:140)
2024-07-23 07:06:07,252 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.selector.ClassLoaderContextSelector.getContext(ClassLoaderContextSelector.java:123)
2024-07-23 07:06:07,253 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.selector.ClassLoaderContextSelector.getContext(ClassLoaderContextSelector.java:117)
2024-07-23 07:06:07,253 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.impl.Log4jContextFactory.getContext(Log4jContextFactory.java:149)
2024-07-23 07:06:07,253 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.core.impl.Log4jContextFactory.getContext(Log4jContextFactory.java:46)
2024-07-23 07:06:07,254 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.LogManager.getContext(LogManager.java:196)
2024-07-23 07:06:07,254 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.logging.log4j.LogManager.getLogger(LogManager.java:599)
2024-07-23 07:06:07,255 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.spark.deploy.yarn.SparkRackResolver.<init>(SparkRackResolver.scala:42)
2024-07-23 07:06:07,255 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.spark.deploy.yarn.SparkRackResolver$.get(SparkRackResolver.scala:114)
2024-07-23 07:06:07,255 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.spark.scheduler.cluster.YarnScheduler.<init>(YarnScheduler.scala:31)
2024-07-23 07:06:07,256 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.spark.scheduler.cluster.YarnClusterScheduler.<init>(YarnClusterScheduler.scala:27)
2024-07-23 07:06:07,256 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.spark.scheduler.cluster.YarnClusterManager.createTaskScheduler(YarnClusterManager.scala:34)
2024-07-23 07:06:07,257 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.spark.SparkContext$.org$apache$spark$SparkContext$$createTaskScheduler(SparkContext.scala:3236)
2024-07-23 07:06:07,257 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.spark.SparkContext.<init>(SparkContext.scala:584)
2024-07-23 07:06:07,257 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.spark.SparkContext.<init>(SparkContext.scala:130)
2024-07-23 07:06:07,258 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:53)
2024-07-23 07:06:07,258 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at io.cdap.cdap.etl.spark.batch.BatchSparkPipelineDriver.run(BatchSparkPipelineDriver.java:192)
2024-07-23 07:06:07,259 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at io.cdap.cdap.app.runtime.spark.SparkMainWrapper$.main(SparkMainWrapper.scala:88)
2024-07-23 07:06:07,259 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at io.cdap.cdap.app.runtime.spark.SparkMainWrapper.main(SparkMainWrapper.scala)
2024-07-23 07:06:07,260 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2024-07-23 07:06:07,260 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2024-07-23 07:06:07,260 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2024-07-23 07:06:07,261 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
2024-07-23 07:06:07,261 - WARN  [Driver:o.a.s.d.y.ApplicationMaster@137] - 	at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:738)
2024-07-23 07:06:07,910 - DEBUG [SparkDriverService:i.c.c.a.r.s.d.SparkDriverService@160] - Heartbeat loop completed with state 'COMPLETED_WITH_EXCEPTION' and terminate timestamp 'null'
2024-07-23 07:06:07,910 - DEBUG [SparkDriverService:i.c.c.a.r.s.d.SparkDriverService@178] - SparkDriverService completed with program completion state 'COMPLETED_WITH_EXCEPTION'
2024-07-23 07:06:07,945 - INFO  [SparkDriverHttpService STOPPING:i.c.h.NettyHttpService@258] - Stopping HTTP Service phase-1-http-service

The pipeline failed due to conflicting org.apache.logging.log4j.log4j-api:2.20.0 dependency which is bundled in application.jar.

Hence, removing the org.apache.logging.log4j from application.jar using HadoopClassExcluder.

Testing

• Run few pipelines on dataproc 2.2 & dataproc 2.1 and they completed successfully.
• Also tested a pipeline using pyspark plugin.