use of io.cdap.cdap.common.lang.CombineClassLoader in project cdap by cdapio.
the class InMemoryConfigurator method createResponse.
private <T extends Config> ConfigResponse createResponse(Application<T> app, ClassLoader artifactClassLoader) throws Exception {
// This Gson cannot be static since it is used to deserialize user class.
// Gson will keep a static map to class, hence will leak the classloader
Gson gson = new GsonBuilder().registerTypeAdapterFactory(new CaseInsensitiveEnumTypeAdapterFactory()).create();
// Now, we call configure, which returns application specification.
DefaultAppConfigurer configurer;
File tempDir = DirUtils.createTempDir(baseUnpackDir);
try (PluginInstantiator pluginInstantiator = new PluginInstantiator(cConf, app.getClass().getClassLoader(), tempDir)) {
RuntimeConfigurer runtimeConfigurer = runtimeInfo != null ? new DefaultAppRuntimeConfigurer(appNamespace.getId(), remoteClientFactory, runtimeInfo.getUserArguments(), runtimeInfo.getExistingAppSpec()) : null;
configurer = new DefaultAppConfigurer(appNamespace, artifactId, app, configString, pluginFinder, pluginInstantiator, runtimeConfigurer, runtimeInfo, featureFlagsProvider);
T appConfig;
Type configType = Artifacts.getConfigType(app.getClass());
if (configString.isEmpty()) {
// noinspection unchecked
appConfig = ((Class<T>) configType).newInstance();
} else {
try {
appConfig = gson.fromJson(configString, configType);
} catch (JsonSyntaxException e) {
throw new IllegalArgumentException("Invalid JSON configuration was provided. Please check the syntax.", e);
}
}
try {
ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(new CombineClassLoader(null, app.getClass().getClassLoader(), getClass().getClassLoader()));
try {
app.configure(configurer, new DefaultApplicationContext<>(appConfig));
} finally {
ClassLoaders.setContextClassLoader(oldClassLoader);
}
} catch (Throwable t) {
Throwable rootCause = Throwables.getRootCause(t);
if (rootCause instanceof ClassNotFoundException) {
// Heuristic to provide better error message
String missingClass = rootCause.getMessage();
// If the missing class has "spark" in the name, try to see if Spark is available
if (missingClass.startsWith("org.apache.spark.") || missingClass.startsWith("io.cdap.cdap.api.spark.")) {
// Try to load the SparkContext class, which should be available if Spark is available in the platform
try {
artifactClassLoader.loadClass("org.apache.spark.SparkContext");
} catch (ClassNotFoundException e) {
// Spark is not available, it is most likely caused by missing Spark in the platform
throw new IllegalStateException("Missing Spark related class " + missingClass + ". It may be caused by unavailability of Spark. " + "Please verify environment variable " + Constants.SPARK_HOME + " is set correctly", t);
}
// Spark is available, can be caused by incompatible Spark version
throw new InvalidArtifactException("Missing Spark related class " + missingClass + ". Configured to use Spark located at " + System.getenv(Constants.SPARK_HOME) + ", which may be incompatible with the one required by the application", t);
}
// then the missing class is most likely due to some missing library in the artifact jar
throw new InvalidArtifactException("Missing class " + missingClass + ". It may be caused by missing dependency jar(s) in the artifact jar.", t);
}
throw t;
}
} finally {
try {
DirUtils.deleteDirectoryContents(tempDir);
} catch (IOException e) {
LOG.warn("Exception raised when deleting directory {}", tempDir, e);
}
}
ApplicationSpecification specification = configurer.createSpecification(applicationName, applicationVersion);
AppSpecInfo appSpecInfo = new AppSpecInfo(specification, configurer.getSystemTables(), configurer.getMetadata());
return new DefaultConfigResponse(0, appSpecInfo);
}
use of io.cdap.cdap.common.lang.CombineClassLoader in project cdap by cdapio.
the class DefaultServicePluginConfigurer method createClassLoader.
@Override
public ClassLoader createClassLoader() {
Map<String, Plugin> plugins = getPlugins().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, v -> v.getValue().getPlugin()));
ClassLoader pluginsClassLoader = PluginClassLoaders.createFilteredPluginsClassLoader(plugins, getPluginInstantiator());
return new CombineClassLoader(null, programClassLoader, pluginsClassLoader, getClass().getClassLoader());
}
use of io.cdap.cdap.common.lang.CombineClassLoader in project cdap by cdapio.
the class DistributedProgramRunner method run.
@Override
public final ProgramController run(final Program program, ProgramOptions oldOptions) {
validateOptions(program, oldOptions);
CConfiguration cConf = CConfiguration.copy(this.cConf);
// Reload config for log extension jar update (CDAP-15091)
cConf.reloadConfiguration();
File tempDir = DirUtils.createTempDir(new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile());
try {
ProgramLaunchConfig launchConfig = new ProgramLaunchConfig();
if (clusterMode == ClusterMode.ISOLATED) {
// For isolated mode, the hadoop classes comes from the hadoop classpath in the target cluster directly
launchConfig.addExtraClasspath(Collections.singletonList("$HADOOP_CLASSPATH"));
}
setupLaunchConfig(launchConfig, program, oldOptions, cConf, hConf, tempDir);
// Add extra localize resources needed by the program runner
final Map<String, LocalizeResource> localizeResources = new HashMap<>(launchConfig.getExtraResources());
final List<String> additionalClassPaths = new ArrayList<>();
addContainerJars(cConf, localizeResources, additionalClassPaths);
addAdditionalLogAppenderJars(cConf, tempDir, localizeResources, SystemArguments.getProfileProvisioner(oldOptions.getArguments().asMap()));
prepareHBaseDDLExecutorResources(tempDir, cConf, localizeResources);
List<URI> configResources = localizeConfigs(createContainerCConf(cConf), createContainerHConf(this.hConf), tempDir, localizeResources);
// Localize the program jar
Location programJarLocation = program.getJarLocation();
final String programJarName = programJarLocation.getName();
localizeResources.put(programJarName, new LocalizeResource(programJarLocation.toURI(), false));
// Localize the app spec
localizeResources.put(APP_SPEC_FILE_NAME, new LocalizeResource(saveJsonFile(program.getApplicationSpecification(), ApplicationSpecification.class, File.createTempFile("appSpec", ".json", tempDir))));
URI logbackURI = getLogBackURI(program);
if (logbackURI != null) {
// Localize the logback xml
localizeResources.put(LOGBACK_FILE_NAME, new LocalizeResource(logbackURI, false));
}
// Update the ProgramOptions to carry program and runtime information necessary to reconstruct the program
// and runs it in the remote container
Map<String, String> extraSystemArgs = new HashMap<>(launchConfig.getExtraSystemArguments());
extraSystemArgs.put(ProgramOptionConstants.PROGRAM_JAR, programJarName);
extraSystemArgs.put(ProgramOptionConstants.HADOOP_CONF_FILE, HADOOP_CONF_FILE_NAME);
extraSystemArgs.put(ProgramOptionConstants.CDAP_CONF_FILE, CDAP_CONF_FILE_NAME);
extraSystemArgs.put(ProgramOptionConstants.APP_SPEC_FILE, APP_SPEC_FILE_NAME);
ProgramOptions options = updateProgramOptions(oldOptions, localizeResources, DirUtils.createTempDir(tempDir), extraSystemArgs);
ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(options));
// Localize the serialized program options
localizeResources.put(PROGRAM_OPTIONS_FILE_NAME, new LocalizeResource(saveJsonFile(options, ProgramOptions.class, File.createTempFile("program.options", ".json", tempDir))));
Callable<ProgramController> callable = () -> {
ProgramTwillApplication twillApplication = new ProgramTwillApplication(programRunId, options, launchConfig.getRunnables(), launchConfig.getLaunchOrder(), localizeResources, createEventHandler(cConf, programRunId, options));
TwillPreparer twillPreparer = twillRunner.prepare(twillApplication);
// Also add the configuration files to container classpath so that the
// TwillAppLifecycleEventHandler can get it. This can be removed when TWILL-246 is fixed.
// Only ON_PREMISE mode will be using EventHandler
twillPreparer.withResources(configResources);
Map<String, String> userArgs = options.getUserArguments().asMap();
// Setup log level
twillPreparer.setLogLevels(transformLogLevels(SystemArguments.getLogLevels(userArgs)));
// Set the configuration for the twill application
Map<String, String> twillConfigs = new HashMap<>();
if (DistributedProgramRunner.this instanceof LongRunningDistributedProgramRunner) {
twillConfigs.put(Configs.Keys.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL, cConf.get(Constants.AppFabric.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL));
} else {
// For non long running program type, set the max attempts to 1 to avoid YARN retry.
// If the AM container dies, the program execution will be marked as failure.
// Note that this setting is only applicable to the Twill YARN application
// (e.g. workflow, Spark client, MR client, etc), but not to the actual Spark / MR job.
twillConfigs.put(Configs.Keys.YARN_MAX_APP_ATTEMPTS, Integer.toString(1));
}
// Add twill configurations coming from the runtime arguments
twillConfigs.putAll(SystemArguments.getTwillApplicationConfigs(userArgs));
twillPreparer.withConfiguration(twillConfigs);
// Setup per runnable configurations
for (Map.Entry<String, RunnableDefinition> entry : launchConfig.getRunnables().entrySet()) {
String runnable = entry.getKey();
RunnableDefinition runnableDefinition = entry.getValue();
if (runnableDefinition.getMaxRetries() != null) {
twillPreparer.withMaxRetries(runnable, runnableDefinition.getMaxRetries());
}
twillPreparer.setLogLevels(runnable, transformLogLevels(runnableDefinition.getLogLevels()));
twillPreparer.withConfiguration(runnable, runnableDefinition.getTwillRunnableConfigs());
// Add cdap-security.xml if using secrets, and set the runnable identity.
if (twillPreparer instanceof SecureTwillPreparer) {
String twillSystemIdentity = cConf.get(Constants.Twill.Security.IDENTITY_SYSTEM);
if (twillSystemIdentity != null) {
SecurityContext securityContext = new SecurityContext.Builder().withIdentity(twillSystemIdentity).build();
twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecurityContext(runnable, securityContext);
}
String securityName = cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_NAME);
String securityPath = cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_PATH);
twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(runnable, new SecretDisk(securityName, securityPath));
}
}
if (options.isDebug()) {
twillPreparer.enableDebugging();
}
logProgramStart(program, options);
// Add scheduler queue name if defined
String schedulerQueueName = options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE);
if (schedulerQueueName != null && !schedulerQueueName.isEmpty()) {
LOG.info("Setting scheduler queue for app {} as {}", program.getId(), schedulerQueueName);
twillPreparer.setSchedulerQueue(schedulerQueueName);
}
// Set JVM options based on configuration
String jvmOpts = cConf.get(Constants.AppFabric.PROGRAM_JVM_OPTS);
if (!Strings.isNullOrEmpty(jvmOpts)) {
twillPreparer.addJVMOptions(jvmOpts);
}
if (logbackURI != null) {
twillPreparer.addJVMOptions("-Dlogback.configurationFile=" + LOGBACK_FILE_NAME);
}
addLogHandler(twillPreparer, cConf);
// Setup the environment for the container logback.xml
twillPreparer.withEnv(Collections.singletonMap("CDAP_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR));
// Add dependencies
Set<Class<?>> extraDependencies = addExtraDependencies(cConf, new HashSet<>(launchConfig.getExtraDependencies()));
twillPreparer.withDependencies(extraDependencies);
// Add the additional classes to the classpath that comes from the container jar setting
twillPreparer.withClassPaths(additionalClassPaths);
twillPreparer.withClassPaths(launchConfig.getExtraClasspath());
twillPreparer.withEnv(launchConfig.getExtraEnv());
// Add the YARN_APPLICATION_CLASSPATH so that yarn classpath are included in the twill container.
// The Yarn app classpath goes last
List<String> yarnAppClassPath = Arrays.asList(hConf.getTrimmedStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
twillPreparer.withApplicationClassPaths(yarnAppClassPath).withClassPaths(yarnAppClassPath);
twillPreparer.withBundlerClassAcceptor(launchConfig.getClassAcceptor()).withApplicationArguments(PROGRAM_OPTIONS_FILE_NAME).setClassLoader(MainClassLoader.class.getName());
TwillController twillController;
// Change the context classloader to the combine classloader of this ProgramRunner and
// all the classloaders of the dependencies classes so that Twill can trace classes.
ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(new CombineClassLoader(DistributedProgramRunner.this.getClass().getClassLoader(), extraDependencies.stream().map(Class::getClassLoader)::iterator));
try {
twillController = twillPreparer.start(cConf.getLong(Constants.AppFabric.PROGRAM_MAX_START_SECONDS), TimeUnit.SECONDS);
// Block on the twill controller until it is in running state or terminated (due to failure)
CountDownLatch latch = new CountDownLatch(1);
twillController.onRunning(latch::countDown, Threads.SAME_THREAD_EXECUTOR);
twillController.onTerminated(latch::countDown, Threads.SAME_THREAD_EXECUTOR);
latch.await(cConf.getLong(Constants.AppFabric.PROGRAM_MAX_START_SECONDS), TimeUnit.SECONDS);
} finally {
ClassLoaders.setContextClassLoader(oldClassLoader);
}
return createProgramController(programRunId, addCleanupListener(twillController, program, tempDir));
};
return impersonator.doAs(programRunId, callable);
} catch (Exception e) {
deleteDirectory(tempDir);
throw Throwables.propagate(e);
}
}
use of io.cdap.cdap.common.lang.CombineClassLoader in project cdap by caskdata.
the class PluginClassLoaders method createFilteredPluginsClassLoader.
/**
* Returns a {@link ClassLoader} that only allows loading of plugin classes and plugin exported classes.
* It should only be used in context when a single ClassLoader is needed to load all different kinds of user classes
* (e.g. in MapReduce/Spark).
*/
public static ClassLoader createFilteredPluginsClassLoader(Map<String, Plugin> plugins, @Nullable PluginInstantiator pluginInstantiator) {
if (plugins.isEmpty() || pluginInstantiator == null) {
return new CombineClassLoader(null);
}
try {
// Gather all explicitly used plugin class names. It is needed for external plugin case.
Multimap<Plugin, String> artifactPluginClasses = getArtifactPluginClasses(plugins);
List<ClassLoader> pluginClassLoaders = new ArrayList<>();
for (Plugin plugin : plugins.values()) {
ClassLoader pluginClassLoader = pluginInstantiator.getPluginClassLoader(plugin);
if (pluginClassLoader instanceof PluginClassLoader) {
// A ClassLoader to allow loading of all plugin classes used by the program.
Collection<String> pluginClasses = artifactPluginClasses.get(plugin);
if (!pluginClasses.isEmpty()) {
pluginClassLoaders.add(createClassFilteredClassLoader(pluginClasses, pluginClassLoader));
}
// A ClassLoader to allow all export package classes to be loadable.
pluginClassLoaders.add(((PluginClassLoader) pluginClassLoader).getExportPackagesClassLoader());
}
}
return new CombineClassLoader(null, pluginClassLoaders);
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
use of io.cdap.cdap.common.lang.CombineClassLoader in project cdap by caskdata.
the class PluginClassLoader method createParent.
static ClassLoader createParent(ClassLoader templateClassLoader) {
// Find the ProgramClassLoader from the template ClassLoader
ClassLoader programClassLoader = templateClassLoader;
while (programClassLoader != null && !(programClassLoader instanceof ProgramClassLoader)) {
programClassLoader = programClassLoader.getParent();
}
// This shouldn't happen
Preconditions.checkArgument(programClassLoader != null, "Cannot find ProgramClassLoader");
// Package filtered classloader of the template classloader, which only classes in "Export-Packages" are loadable.
Manifest manifest = ((ProgramClassLoader) programClassLoader).getManifest();
Set<String> exportPackages = ManifestFields.getExportPackages(manifest);
ClassLoader filteredTemplateClassLoader = new PackageFilterClassLoader(templateClassLoader, exportPackages::contains);
// followed by template export-packages, then by a plugin lib jars.
return new CombineClassLoader(programClassLoader.getParent(), filteredTemplateClassLoader);
}
Aggregations