use of co.cask.cdap.internal.app.runtime.plugin.PluginInstantiator in project cdap by caskdata.
the class ArtifactInspector method inspectArtifact.
/**
* Inspect the given artifact to determine the classes contained in the artifact.
*
* @param artifactId the id of the artifact to inspect
* @param artifactFile the artifact file
* @param parentClassLoader the parent classloader to use when inspecting plugins contained in the artifact.
* For example, a ProgramClassLoader created from the artifact the input artifact extends
* @return metadata about the classes contained in the artifact
* @throws IOException if there was an exception opening the jar file
* @throws InvalidArtifactException if the artifact is invalid. For example, if the application main class is not
* actually an Application.
*/
ArtifactClasses inspectArtifact(Id.Artifact artifactId, File artifactFile, @Nullable ClassLoader parentClassLoader) throws IOException, InvalidArtifactException {
Path tmpDir = Paths.get(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).toAbsolutePath();
Files.createDirectories(tmpDir);
Location artifactLocation = Locations.toLocation(artifactFile);
Path stageDir = Files.createTempDirectory(tmpDir, artifactFile.getName());
try {
File unpackedDir = BundleJarUtil.unJar(artifactLocation, Files.createTempDirectory(stageDir, "unpacked-").toFile());
try (CloseableClassLoader artifactClassLoader = artifactClassLoaderFactory.createClassLoader(unpackedDir)) {
ArtifactClasses.Builder builder = inspectApplications(artifactId, ArtifactClasses.builder(), artifactLocation, artifactClassLoader);
try (PluginInstantiator pluginInstantiator = new PluginInstantiator(cConf, parentClassLoader == null ? artifactClassLoader : parentClassLoader, Files.createTempDirectory(stageDir, "plugins-").toFile())) {
pluginInstantiator.addArtifact(artifactLocation, artifactId.toArtifactId());
inspectPlugins(builder, artifactFile, artifactId.toArtifactId(), pluginInstantiator);
}
return builder.build();
}
} catch (EOFException | ZipException e) {
throw new InvalidArtifactException("Artifact " + artifactId + " is not a valid zip file.", e);
} finally {
try {
DirUtils.deleteDirectoryContents(stageDir.toFile());
} catch (IOException e) {
LOG.warn("Exception raised while deleting directory {}", stageDir, e);
}
}
}
use of co.cask.cdap.internal.app.runtime.plugin.PluginInstantiator in project cdap by caskdata.
the class InMemoryConfigurator method getSpecJson.
private <T extends Config> String getSpecJson(Application<T> app) throws Exception {
// This Gson cannot be static since it is used to deserialize user class.
// Gson will keep a static map to class, hence will leak the classloader
Gson gson = new GsonBuilder().registerTypeAdapterFactory(new CaseInsensitiveEnumTypeAdapterFactory()).create();
// Now, we call configure, which returns application specification.
DefaultAppConfigurer configurer;
File tempDir = DirUtils.createTempDir(baseUnpackDir);
try (PluginInstantiator pluginInstantiator = new PluginInstantiator(cConf, app.getClass().getClassLoader(), tempDir)) {
configurer = new DefaultAppConfigurer(appNamespace, artifactId, app, configString, artifactRepository, pluginInstantiator);
T appConfig;
Type configType = Artifacts.getConfigType(app.getClass());
if (configString.isEmpty()) {
//noinspection unchecked
appConfig = ((Class<T>) configType).newInstance();
} else {
try {
appConfig = gson.fromJson(configString, configType);
} catch (JsonSyntaxException e) {
throw new IllegalArgumentException("Invalid JSON configuration was provided. Please check the syntax.", e);
}
}
try {
ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(new CombineClassLoader(null, Arrays.asList(app.getClass().getClassLoader(), getClass().getClassLoader())));
try {
app.configure(configurer, new DefaultApplicationContext<>(appConfig));
} finally {
ClassLoaders.setContextClassLoader(oldClassLoader);
}
} catch (Throwable t) {
Throwable rootCause = Throwables.getRootCause(t);
if (rootCause instanceof ClassNotFoundException) {
// Heuristic to provide better error message
String missingClass = rootCause.getMessage();
// If the missing class has "spark" in the name, try to see if Spark is available
if (missingClass.startsWith("org.apache.spark.") || missingClass.startsWith("co.cask.cdap.api.spark.")) {
// Try to load the SparkContext class, which should be available if Spark is available in the platform
try {
artifactClassLoader.loadClass("org.apache.spark.SparkContext");
} catch (ClassNotFoundException e) {
// Spark is not available, it is most likely caused by missing Spark in the platform
throw new IllegalStateException("Missing Spark related class " + missingClass + ". It may be caused by unavailability of Spark. " + "Please verify environment variable " + Constants.SPARK_HOME + " is set correctly", t);
}
// Spark is available, can be caused by incompatible Spark version
throw new InvalidArtifactException("Missing Spark related class " + missingClass + ". Configured to use Spark located at " + System.getenv(Constants.SPARK_HOME) + ", which may be incompatible with the one required by the application", t);
}
// then the missing class is most likely due to some missing library in the artifact jar
throw new InvalidArtifactException("Missing class " + missingClass + ". It may be caused by missing dependency jar(s) in the artifact jar.", t);
}
throw t;
}
} finally {
try {
DirUtils.deleteDirectoryContents(tempDir);
} catch (IOException e) {
LOG.warn("Exception raised when deleting directory {}", tempDir, e);
}
}
ApplicationSpecification specification = configurer.createSpecification(applicationName, applicationVersion);
// TODO: The SchemaGenerator should be injected
return ApplicationSpecificationAdapter.create(new ReflectionSchemaGenerator()).toJson(specification);
}
use of co.cask.cdap.internal.app.runtime.plugin.PluginInstantiator in project cdap by caskdata.
the class WorkerProgramRunner method run.
@Override
public ProgramController run(Program program, ProgramOptions options) {
ApplicationSpecification appSpec = program.getApplicationSpecification();
Preconditions.checkNotNull(appSpec, "Missing application specification.");
int instanceId = Integer.parseInt(options.getArguments().getOption(ProgramOptionConstants.INSTANCE_ID, "-1"));
Preconditions.checkArgument(instanceId >= 0, "Missing instance Id");
int instanceCount = Integer.parseInt(options.getArguments().getOption(ProgramOptionConstants.INSTANCES, "0"));
Preconditions.checkArgument(instanceCount > 0, "Invalid or missing instance count");
RunId runId = ProgramRunners.getRunId(options);
ProgramType programType = program.getType();
Preconditions.checkNotNull(programType, "Missing processor type.");
Preconditions.checkArgument(programType == ProgramType.WORKER, "Only Worker process type is supported.");
WorkerSpecification workerSpec = appSpec.getWorkers().get(program.getName());
Preconditions.checkArgument(workerSpec != null, "Missing Worker specification for %s", program.getId());
String instances = options.getArguments().getOption(ProgramOptionConstants.INSTANCES, String.valueOf(workerSpec.getInstances()));
WorkerSpecification newWorkerSpec = new WorkerSpecification(workerSpec.getClassName(), workerSpec.getName(), workerSpec.getDescription(), workerSpec.getProperties(), workerSpec.getDatasets(), workerSpec.getResources(), Integer.valueOf(instances));
// Setup dataset framework context, if required
if (datasetFramework instanceof ProgramContextAware) {
ProgramId programId = program.getId();
((ProgramContextAware) datasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
}
final PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
try {
BasicWorkerContext context = new BasicWorkerContext(newWorkerSpec, program, options, cConf, instanceId, instanceCount, metricsCollectionService, datasetFramework, txClient, discoveryServiceClient, streamWriterFactory, pluginInstantiator, secureStore, secureStoreManager, messagingService);
WorkerDriver worker = new WorkerDriver(program, newWorkerSpec, context);
// Add a service listener to make sure the plugin instantiator is closed when the worker driver finished.
worker.addListener(new ServiceListenerAdapter() {
@Override
public void terminated(Service.State from) {
Closeables.closeQuietly(pluginInstantiator);
}
@Override
public void failed(Service.State from, Throwable failure) {
Closeables.closeQuietly(pluginInstantiator);
}
}, Threads.SAME_THREAD_EXECUTOR);
ProgramController controller = new WorkerControllerServiceAdapter(worker, program.getId(), runId, workerSpec.getName() + "-" + instanceId);
worker.start();
return controller;
} catch (Throwable t) {
Closeables.closeQuietly(pluginInstantiator);
throw t;
}
}
use of co.cask.cdap.internal.app.runtime.plugin.PluginInstantiator in project cdap by caskdata.
the class DefaultAppConfigurer method addSpark.
@Override
public void addSpark(Spark spark) {
Preconditions.checkArgument(spark != null, "Spark cannot be null.");
DefaultSparkConfigurer configurer = null;
// It is a bit hacky here to look for the DefaultExtendedSparkConfigurer implementation through the
// SparkRunnerClassloader directly (CDAP-11797)
ClassLoader sparkRunnerClassLoader = ClassLoaders.findByName(spark.getClass().getClassLoader(), "co.cask.cdap.app.runtime.spark.classloader.SparkRunnerClassLoader");
if (sparkRunnerClassLoader != null) {
try {
configurer = (DefaultSparkConfigurer) sparkRunnerClassLoader.loadClass("co.cask.cdap.app.deploy.spark.DefaultExtendedSparkConfigurer").getConstructor(Spark.class, Id.Namespace.class, Id.Artifact.class, ArtifactRepository.class, PluginInstantiator.class).newInstance(spark, deployNamespace, artifactId, artifactRepository, pluginInstantiator);
} catch (Exception e) {
// Ignore it and the configurer will be defaulted to DefaultSparkConfigurer
LOG.trace("No DefaultExtendedSparkConfigurer found. Fallback to DefaultSparkConfigurer.", e);
}
}
if (configurer == null) {
configurer = new DefaultSparkConfigurer(spark, deployNamespace, artifactId, artifactRepository, pluginInstantiator);
}
spark.configure(configurer);
addDatasetsAndPlugins(configurer);
SparkSpecification spec = configurer.createSpecification();
sparks.put(spec.getName(), spec);
}
use of co.cask.cdap.internal.app.runtime.plugin.PluginInstantiator in project cdap by caskdata.
the class ArtifactRepositoryTest method testPluginSelector.
@Test
public void testPluginSelector() throws Exception {
// No plugin yet
ArtifactRange appArtifactRange = new ArtifactRange(APP_ARTIFACT_ID.getNamespace().getId(), APP_ARTIFACT_ID.getName(), APP_ARTIFACT_ID.getVersion(), true, APP_ARTIFACT_ID.getVersion(), true);
try {
artifactRepository.findPlugin(NamespaceId.DEFAULT, appArtifactRange, "plugin", "TestPlugin2", new PluginSelector());
Assert.fail();
} catch (PluginNotExistsException e) {
// expected
}
File pluginDir = DirUtils.createTempDir(tmpDir);
// Create a plugin jar. It contains two plugins, TestPlugin and TestPlugin2 inside.
Id.Artifact artifact1Id = Id.Artifact.from(Id.Namespace.DEFAULT, "myPlugin", "1.0");
Manifest manifest = createManifest(ManifestFields.EXPORT_PACKAGE, TestPlugin.class.getPackage().getName());
File jarFile = createPluginJar(TestPlugin.class, new File(tmpDir, "myPlugin-1.0.jar"), manifest);
// Build up the plugin repository.
Set<ArtifactRange> parents = ImmutableSet.of(new ArtifactRange(APP_ARTIFACT_ID.getNamespace().getId(), APP_ARTIFACT_ID.getName(), new ArtifactVersion("1.0.0"), new ArtifactVersion("2.0.0")));
artifactRepository.addArtifact(artifact1Id, jarFile, parents);
// Should get the only version.
Map.Entry<ArtifactDescriptor, PluginClass> plugin = artifactRepository.findPlugin(NamespaceId.DEFAULT, appArtifactRange, "plugin", "TestPlugin2", new PluginSelector());
Assert.assertNotNull(plugin);
Assert.assertEquals(new ArtifactVersion("1.0"), plugin.getKey().getArtifactId().getVersion());
Assert.assertEquals("TestPlugin2", plugin.getValue().getName());
Files.copy(Locations.newInputSupplier(plugin.getKey().getLocation()), new File(pluginDir, Artifacts.getFileName(plugin.getKey().getArtifactId())));
// Create another plugin jar with later version and update the repository
Id.Artifact artifact2Id = Id.Artifact.from(Id.Namespace.DEFAULT, "myPlugin", "2.0");
jarFile = createPluginJar(TestPlugin.class, new File(tmpDir, "myPlugin-2.0.jar"), manifest);
artifactRepository.addArtifact(artifact2Id, jarFile, parents);
// Should select the latest version
plugin = artifactRepository.findPlugin(NamespaceId.DEFAULT, appArtifactRange, "plugin", "TestPlugin2", new PluginSelector());
Assert.assertNotNull(plugin);
Assert.assertEquals(new ArtifactVersion("2.0"), plugin.getKey().getArtifactId().getVersion());
Assert.assertEquals("TestPlugin2", plugin.getValue().getName());
Files.copy(Locations.newInputSupplier(plugin.getKey().getLocation()), new File(pluginDir, Artifacts.getFileName(plugin.getKey().getArtifactId())));
// Load the Plugin class from the classLoader.
try (PluginInstantiator instantiator = new PluginInstantiator(cConf, appClassLoader, pluginDir)) {
ClassLoader pluginClassLoader = instantiator.getArtifactClassLoader(plugin.getKey().getArtifactId());
Class<?> pluginClass = pluginClassLoader.loadClass(TestPlugin2.class.getName());
// Use a custom plugin selector to select with smallest version
plugin = artifactRepository.findPlugin(NamespaceId.DEFAULT, appArtifactRange, "plugin", "TestPlugin2", new PluginSelector() {
@Override
public Map.Entry<ArtifactId, PluginClass> select(SortedMap<ArtifactId, PluginClass> plugins) {
return plugins.entrySet().iterator().next();
}
});
Assert.assertNotNull(plugin);
Assert.assertEquals(new ArtifactVersion("1.0"), plugin.getKey().getArtifactId().getVersion());
Assert.assertEquals("TestPlugin2", plugin.getValue().getName());
// Load the Plugin class again from the current plugin selected
// The plugin class should be different (from different ClassLoader)
// The empty class should be the same (from the plugin lib ClassLoader)
pluginClassLoader = instantiator.getArtifactClassLoader(plugin.getKey().getArtifactId());
Assert.assertNotSame(pluginClass, pluginClassLoader.loadClass(TestPlugin2.class.getName()));
// From the pluginClassLoader, loading export classes from the template jar should be allowed
Class<?> cls = pluginClassLoader.loadClass(PluginTestRunnable.class.getName());
// The class should be loaded from the parent artifact's classloader
Assert.assertSame(appClassLoader.loadClass(PluginTestRunnable.class.getName()), cls);
// From the plugin classloader, all cdap api classes is loadable as well.
cls = pluginClassLoader.loadClass(Application.class.getName());
// The Application class should be the same as the one in the system classloader
Assert.assertSame(Application.class, cls);
}
}
Aggregations