From 4bc543d3db33956e9ce48a218e6dc636019e0780 Mon Sep 17 00:00:00 2001 From: albertshau Date: Wed, 30 Aug 2023 14:58:51 -0700 Subject: [PATCH] CDAP-20791 fix master service startup on Hadoop Move hconf and cconf localization earlier so they are properly applied before creating the TwillPreparer. --- .../cdap/data/runtime/main/MasterServiceMain.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java index f84dc6a8c6d4..02d7d156b4f0 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java @@ -936,7 +936,12 @@ private TwillController startTwillApplication(TwillRunnerService twillRunner, try { MasterTwillApplication masterTwillApp = new MasterTwillApplication(cConf, getServiceInstances(serviceStore, cConf)); + List extraClassPath = masterTwillApp.prepareLocalizeResource(runDir, hConf); TwillPreparer preparer = twillRunner.prepare(masterTwillApp); + // Setup extra classpath. Currently twill doesn't support different classpath per runnable, + // hence we just set it for all containers. The actual jars are localized via the MasterTwillApplication, + // and having missing jars as specified in the classpath is ok. + preparer = preparer.withClassPaths(extraClassPath); Map twillConfigs = new HashMap<>(); if (!cConf.getBoolean(Constants.COLLECT_CONTAINER_LOGS)) { @@ -979,12 +984,6 @@ private TwillController startTwillApplication(TwillRunnerService twillRunner, preparer.withApplicationClassPaths(yarnAppClassPath) .withBundlerClassAcceptor(new HadoopClassExcluder()); - // Setup extra classpath. Currently twill doesn't support different classpath per runnable, - // hence we just set it for all containers. The actual jars are localized via the MasterTwillApplication, - // and having missing jars as specified in the classpath is ok. - List extraClassPath = masterTwillApp.prepareLocalizeResource(runDir, hConf); - preparer = preparer.withClassPaths(extraClassPath); - // Set the container to use MasterServiceMainClassLoader for class rewriting preparer.setClassLoader(MasterServiceMainClassLoader.class.getName());