Examples with PluginInstantiator io.cdap.cdap.internal.app.runtime.plugin.PluginInstantiator used on opensource projects

Example 1 with PluginInstantiator

use of io.cdap.cdap.internal.app.runtime.plugin.PluginInstantiator in project cdap by caskdata.

the class DefaultAppConfigurer method addSpark.

@Override
public void addSpark(Spark spark) {
    Preconditions.checkArgument(spark != null, "Spark cannot be null.");
    DefaultSparkConfigurer configurer = null;
    // It is a bit hacky here to look for the DefaultExtendedSparkConfigurer implementation through the
    // SparkRunnerClassloader directly (CDAP-11797)
    ClassLoader sparkRunnerClassLoader = ClassLoaders.findByName(spark.getClass().getClassLoader(), "io.cdap.cdap.app.runtime.spark.classloader.SparkRunnerClassLoader");
    if (sparkRunnerClassLoader != null) {
        try {
            configurer = (DefaultSparkConfigurer) sparkRunnerClassLoader.loadClass("io.cdap.cdap.app.deploy.spark.DefaultExtendedSparkConfigurer").getConstructor(Spark.class, Id.Namespace.class, Id.Artifact.class, PluginFinder.class, PluginInstantiator.class, AppDeploymentRuntimeInfo.class, FeatureFlagsProvider.class).newInstance(spark, deployNamespace, artifactId, pluginFinder, pluginInstantiator, runtimeInfo, getFeatureFlagsProvider());
        } catch (Exception e) {
            // Ignore it and the configurer will be defaulted to DefaultSparkConfigurer
            LOG.trace("No DefaultExtendedSparkConfigurer found. Fallback to DefaultSparkConfigurer.", e);
        }
    }
    if (configurer == null) {
        configurer = new DefaultSparkConfigurer(spark, deployNamespace, artifactId, pluginFinder, pluginInstantiator, runtimeInfo, getFeatureFlagsProvider());
    }
    spark.configure(configurer);
    addDatasetsAndPlugins(configurer);
    SparkSpecification spec = configurer.createSpecification();
    sparks.put(spec.getName(), spec);
}

Example 2 with PluginInstantiator

use of io.cdap.cdap.internal.app.runtime.plugin.PluginInstantiator in project cdap by caskdata.

the class InMemoryConfigurator method createResponse.

private <T extends Config> ConfigResponse createResponse(Application<T> app, ClassLoader artifactClassLoader) throws Exception {
    // This Gson cannot be static since it is used to deserialize user class.
    // Gson will keep a static map to class, hence will leak the classloader
    Gson gson = new GsonBuilder().registerTypeAdapterFactory(new CaseInsensitiveEnumTypeAdapterFactory()).create();
    // Now, we call configure, which returns application specification.
    DefaultAppConfigurer configurer;
    File tempDir = DirUtils.createTempDir(baseUnpackDir);
    try (PluginInstantiator pluginInstantiator = new PluginInstantiator(cConf, app.getClass().getClassLoader(), tempDir)) {
        RuntimeConfigurer runtimeConfigurer = runtimeInfo != null ? new DefaultAppRuntimeConfigurer(appNamespace.getId(), remoteClientFactory, runtimeInfo.getUserArguments(), runtimeInfo.getExistingAppSpec()) : null;
        configurer = new DefaultAppConfigurer(appNamespace, artifactId, app, configString, pluginFinder, pluginInstantiator, runtimeConfigurer, runtimeInfo, featureFlagsProvider);
        T appConfig;
        Type configType = Artifacts.getConfigType(app.getClass());
        if (configString.isEmpty()) {
            // noinspection unchecked
            appConfig = ((Class<T>) configType).newInstance();
        } else {
            try {
                appConfig = gson.fromJson(configString, configType);
            } catch (JsonSyntaxException e) {
                throw new IllegalArgumentException("Invalid JSON configuration was provided. Please check the syntax.", e);
            }
        }
        try {
            ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(new CombineClassLoader(null, app.getClass().getClassLoader(), getClass().getClassLoader()));
            try {
                app.configure(configurer, new DefaultApplicationContext<>(appConfig));
            } finally {
                ClassLoaders.setContextClassLoader(oldClassLoader);
            }
        } catch (Throwable t) {
            Throwable rootCause = Throwables.getRootCause(t);
            if (rootCause instanceof ClassNotFoundException) {
                // Heuristic to provide better error message
                String missingClass = rootCause.getMessage();
                // If the missing class has "spark" in the name, try to see if Spark is available
                if (missingClass.startsWith("org.apache.spark.") || missingClass.startsWith("io.cdap.cdap.api.spark.")) {
                    // Try to load the SparkContext class, which should be available if Spark is available in the platform
                    try {
                        artifactClassLoader.loadClass("org.apache.spark.SparkContext");
                    } catch (ClassNotFoundException e) {
                        // Spark is not available, it is most likely caused by missing Spark in the platform
                        throw new IllegalStateException("Missing Spark related class " + missingClass + ". It may be caused by unavailability of Spark. " + "Please verify environment variable " + Constants.SPARK_HOME + " is set correctly", t);
                    }
                    // Spark is available, can be caused by incompatible Spark version
                    throw new InvalidArtifactException("Missing Spark related class " + missingClass + ". Configured to use Spark located at " + System.getenv(Constants.SPARK_HOME) + ", which may be incompatible with the one required by the application", t);
                }
                // then the missing class is most likely due to some missing library in the artifact jar
                throw new InvalidArtifactException("Missing class " + missingClass + ". It may be caused by missing dependency jar(s) in the artifact jar.", t);
            }
            throw t;
        }
    } finally {
        try {
            DirUtils.deleteDirectoryContents(tempDir);
        } catch (IOException e) {
            LOG.warn("Exception raised when deleting directory {}", tempDir, e);
        }
    }
    ApplicationSpecification specification = configurer.createSpecification(applicationName, applicationVersion);
    AppSpecInfo appSpecInfo = new AppSpecInfo(specification, configurer.getSystemTables(), configurer.getMetadata());
    return new DefaultConfigResponse(0, appSpecInfo);
}

Example 3 with PluginInstantiator

use of io.cdap.cdap.internal.app.runtime.plugin.PluginInstantiator in project cdap by caskdata.

the class ServiceProgramRunner method run.

@Override
public ProgramController run(Program program, ProgramOptions options) {
    int instanceId = Integer.parseInt(options.getArguments().getOption(ProgramOptionConstants.INSTANCE_ID, "-1"));
    Preconditions.checkArgument(instanceId >= 0, "Missing instance Id");
    int instanceCount = Integer.parseInt(options.getArguments().getOption(ProgramOptionConstants.INSTANCES, "0"));
    Preconditions.checkArgument(instanceCount > 0, "Invalid or missing instance count");
    RunId runId = ProgramRunners.getRunId(options);
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType programType = program.getType();
    Preconditions.checkNotNull(programType, "Missing processor type.");
    Preconditions.checkArgument(programType == ProgramType.SERVICE, "Only Service process type is supported.");
    ServiceSpecification spec = appSpec.getServices().get(program.getName());
    String host = options.getArguments().getOption(ProgramOptionConstants.HOST);
    Preconditions.checkArgument(host != null, "No hostname is provided");
    // Setup dataset framework context, if required
    if (datasetFramework instanceof ProgramContextAware) {
        ProgramId programId = program.getId();
        ((ProgramContextAware) datasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
    }
    final PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
    try {
        RetryStrategy retryStrategy = SystemArguments.getRetryStrategy(options.getUserArguments().asMap(), program.getType(), cConf);
        ArtifactManager artifactManager = artifactManagerFactory.create(program.getId().getNamespaceId(), retryStrategy);
        ServiceHttpServer component = new ServiceHttpServer(host, program, options, cConf, spec, instanceId, instanceCount, serviceAnnouncer, metricsCollectionService, datasetFramework, txClient, discoveryServiceClient, pluginInstantiator, secureStore, secureStoreManager, messagingService, artifactManager, metadataReader, metadataPublisher, namespaceQueryAdmin, pluginFinder, fieldLineageWriter, transactionRunner, preferencesFetcher, remoteClientFactory, contextAccessEnforcer);
        // Add a service listener to make sure the plugin instantiator is closed when the http server is finished.
        component.addListener(createRuntimeServiceListener(Collections.singleton(pluginInstantiator)), Threads.SAME_THREAD_EXECUTOR);
        ProgramController controller = new ServiceProgramControllerAdapter(component, program.getId().run(runId));
        component.start();
        return controller;
    } catch (Throwable t) {
        Closeables.closeQuietly(pluginInstantiator);
        throw t;
    }
}

Example 4 with PluginInstantiator

use of io.cdap.cdap.internal.app.runtime.plugin.PluginInstantiator in project cdap by caskdata.

the class DefaultArtifactInspector method inspectArtifact.

/**
 * Inspect the given artifact to determine the classes contained in the artifact.
 *
 * @param artifactId the id of the artifact to inspect
 * @param artifactFile the artifact file
 * @param parentDescriptor {@link ArtifactDescriptor} of parent and grandparent (if any) artifacts.
 * @param additionalPlugins Additional plugin classes
 * @return metadata about the classes contained in the artifact
 * @throws IOException              if there was an exception opening the jar file
 * @throws InvalidArtifactException if the artifact is invalid. For example, if the application main class is not
 *                                  actually an Application.
 */
@Override
public ArtifactClassesWithMetadata inspectArtifact(Id.Artifact artifactId, File artifactFile, List<ArtifactDescriptor> parentDescriptor, Set<PluginClass> additionalPlugins) throws IOException, InvalidArtifactException {
    Path tmpDir = Paths.get(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).toAbsolutePath();
    Files.createDirectories(tmpDir);
    Location artifactLocation = Locations.toLocation(artifactFile);
    EntityImpersonator entityImpersonator = new EntityImpersonator(artifactId.toEntityId(), impersonator);
    Path stageDir = Files.createTempDirectory(tmpDir, artifactFile.getName());
    try (ClassLoaderFolder clFolder = BundleJarUtil.prepareClassLoaderFolder(artifactLocation, () -> Files.createTempDirectory(stageDir, "unpacked-").toFile());
        CloseableClassLoader parentClassLoader = createParentClassLoader(parentDescriptor, entityImpersonator);
        CloseableClassLoader artifactClassLoader = artifactClassLoaderFactory.createClassLoader(clFolder.getDir());
        PluginInstantiator pluginInstantiator = new PluginInstantiator(cConf, parentClassLoader == null ? artifactClassLoader : parentClassLoader, Files.createTempDirectory(stageDir, "plugins-").toFile(), false)) {
        pluginInstantiator.addArtifact(artifactLocation, artifactId.toArtifactId());
        ArtifactClasses.Builder builder = inspectApplications(artifactId, ArtifactClasses.builder(), artifactLocation, artifactClassLoader);
        List<MetadataMutation> mutations = new ArrayList<>();
        inspectPlugins(builder, artifactFile, artifactId.toEntityId(), pluginInstantiator, additionalPlugins, mutations);
        return new ArtifactClassesWithMetadata(builder.build(), mutations);
    } catch (EOFException | ZipException e) {
        throw new InvalidArtifactException("Artifact " + artifactId + " is not a valid zip file.", e);
    } finally {
        try {
            DirUtils.deleteDirectoryContents(stageDir.toFile());
        } catch (IOException e) {
            LOG.warn("Exception raised while deleting directory {}", stageDir, e);
        }
    }
}

Example 5 with PluginInstantiator

use of io.cdap.cdap.internal.app.runtime.plugin.PluginInstantiator in project cdap by caskdata.

the class ArtifactRepositoryTest method testPluginConfigWithNoStringValues.

@Test
public void testPluginConfigWithNoStringValues() throws Exception {
    File pluginDir = TMP_FOLDER.newFolder();
    addPluginArtifact();
    SortedMap<ArtifactDescriptor, Set<PluginClass>> plugins = getPlugins();
    copyArtifacts(pluginDir, plugins);
    String numericValue = "42";
    // Instantiate the plugins and execute them
    try (PluginInstantiator instantiator = new PluginInstantiator(cConf, appClassLoader, pluginDir)) {
        for (Map.Entry<ArtifactDescriptor, Set<PluginClass>> entry : plugins.entrySet()) {
            for (PluginClass pluginClass : entry.getValue()) {
                Plugin pluginInfo = new Plugin(new ArrayList<>(), entry.getKey().getArtifactId(), pluginClass, PluginProperties.builder().add("class.name", TEST_EMPTY_CLASS).add("nullableLongFlag", numericValue).add("host", "example.com").add("aBoolean", "${aBoolean}").add("aByte", numericValue).add("aChar", "${aChar}").add("aDouble", "${aDouble}").add("anInt", numericValue).add("aFloat", "${aFloat}").add("aLong", numericValue).add("aShort", numericValue).build());
                // first test with quotes ("42")
                String jsonPluginStr = GSON.toJson(pluginInfo);
                pluginInfo = GSON.fromJson(jsonPluginStr, Plugin.class);
                instantiator.newInstance(pluginInfo);
                // test without quotes (42)
                pluginInfo = GSON.fromJson(jsonPluginStr.replaceAll("\"" + numericValue + "\"", numericValue), Plugin.class);
                instantiator.newInstance(pluginInfo);
                // test with quotes and dot ("42.0")
                pluginInfo = GSON.fromJson(jsonPluginStr.replaceAll(numericValue, numericValue + ".0"), Plugin.class);
                instantiator.newInstance(pluginInfo);
                // test with dot (42.0)
                pluginInfo = GSON.fromJson(jsonPluginStr.replaceAll("\"" + numericValue + "\"", numericValue + ".0"), Plugin.class);
                instantiator.newInstance(pluginInfo);
                // test with some actual double number 42.5
                pluginInfo = GSON.fromJson(jsonPluginStr.replaceAll("\"" + numericValue + "\"", numericValue + ".5"), Plugin.class);
                try {
                    instantiator.newInstance(pluginInfo);
                    Assert.fail("Plugin instantiation should fail with value '42.5'");
                } catch (InvalidPluginConfigException e) {
                // expected
                }
            }
        }
    }
}