Examples with CombineClassLoader io.cdap.cdap.common.lang.CombineClassLoader used on opensource projects

Example 1 with CombineClassLoader

use of io.cdap.cdap.common.lang.CombineClassLoader in project cdap by caskdata.

the class InMemoryConfigurator method createResponse.

private <T extends Config> ConfigResponse createResponse(Application<T> app, ClassLoader artifactClassLoader) throws Exception {
    // This Gson cannot be static since it is used to deserialize user class.
    // Gson will keep a static map to class, hence will leak the classloader
    Gson gson = new GsonBuilder().registerTypeAdapterFactory(new CaseInsensitiveEnumTypeAdapterFactory()).create();
    // Now, we call configure, which returns application specification.
    DefaultAppConfigurer configurer;
    File tempDir = DirUtils.createTempDir(baseUnpackDir);
    try (PluginInstantiator pluginInstantiator = new PluginInstantiator(cConf, app.getClass().getClassLoader(), tempDir)) {
        RuntimeConfigurer runtimeConfigurer = runtimeInfo != null ? new DefaultAppRuntimeConfigurer(appNamespace.getId(), remoteClientFactory, runtimeInfo.getUserArguments(), runtimeInfo.getExistingAppSpec()) : null;
        configurer = new DefaultAppConfigurer(appNamespace, artifactId, app, configString, pluginFinder, pluginInstantiator, runtimeConfigurer, runtimeInfo, featureFlagsProvider);
        T appConfig;
        Type configType = Artifacts.getConfigType(app.getClass());
        if (configString.isEmpty()) {
            // noinspection unchecked
            appConfig = ((Class<T>) configType).newInstance();
        } else {
            try {
                appConfig = gson.fromJson(configString, configType);
            } catch (JsonSyntaxException e) {
                throw new IllegalArgumentException("Invalid JSON configuration was provided. Please check the syntax.", e);
            }
        }
        try {
            ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(new CombineClassLoader(null, app.getClass().getClassLoader(), getClass().getClassLoader()));
            try {
                app.configure(configurer, new DefaultApplicationContext<>(appConfig));
            } finally {
                ClassLoaders.setContextClassLoader(oldClassLoader);
            }
        } catch (Throwable t) {
            Throwable rootCause = Throwables.getRootCause(t);
            if (rootCause instanceof ClassNotFoundException) {
                // Heuristic to provide better error message
                String missingClass = rootCause.getMessage();
                // If the missing class has "spark" in the name, try to see if Spark is available
                if (missingClass.startsWith("org.apache.spark.") || missingClass.startsWith("io.cdap.cdap.api.spark.")) {
                    // Try to load the SparkContext class, which should be available if Spark is available in the platform
                    try {
                        artifactClassLoader.loadClass("org.apache.spark.SparkContext");
                    } catch (ClassNotFoundException e) {
                        // Spark is not available, it is most likely caused by missing Spark in the platform
                        throw new IllegalStateException("Missing Spark related class " + missingClass + ". It may be caused by unavailability of Spark. " + "Please verify environment variable " + Constants.SPARK_HOME + " is set correctly", t);
                    }
                    // Spark is available, can be caused by incompatible Spark version
                    throw new InvalidArtifactException("Missing Spark related class " + missingClass + ". Configured to use Spark located at " + System.getenv(Constants.SPARK_HOME) + ", which may be incompatible with the one required by the application", t);
                }
                // then the missing class is most likely due to some missing library in the artifact jar
                throw new InvalidArtifactException("Missing class " + missingClass + ". It may be caused by missing dependency jar(s) in the artifact jar.", t);
            }
            throw t;
        }
    } finally {
        try {
            DirUtils.deleteDirectoryContents(tempDir);
        } catch (IOException e) {
            LOG.warn("Exception raised when deleting directory {}", tempDir, e);
        }
    }
    ApplicationSpecification specification = configurer.createSpecification(applicationName, applicationVersion);
    AppSpecInfo appSpecInfo = new AppSpecInfo(specification, configurer.getSystemTables(), configurer.getMetadata());
    return new DefaultConfigResponse(0, appSpecInfo);
}

Example 2 with CombineClassLoader

use of io.cdap.cdap.common.lang.CombineClassLoader in project cdap by caskdata.

the class DefaultServicePluginConfigurer method createClassLoader.

@Override
public ClassLoader createClassLoader() {
    Map<String, Plugin> plugins = getPlugins().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, v -> v.getValue().getPlugin()));
    ClassLoader pluginsClassLoader = PluginClassLoaders.createFilteredPluginsClassLoader(plugins, getPluginInstantiator());
    return new CombineClassLoader(null, programClassLoader, pluginsClassLoader, getClass().getClassLoader());
}

Example 3 with CombineClassLoader

use of io.cdap.cdap.common.lang.CombineClassLoader in project cdap by caskdata.

the class MultipleOutputs method getRecordWriter.

// by being synchronized MultipleOutputTask can be use with a MultithreadedMapper.
@SuppressWarnings("unchecked")
private synchronized RecordWriter getRecordWriter(String namedOutput) throws IOException, InterruptedException {
    // look for record-writer in the cache
    RecordWriter writer = recordWriters.get(namedOutput);
    // If not in cache, create a new one
    if (writer == null) {
        // get the record writer from context output format
        TaskAttemptContext taskContext = getContext(namedOutput);
        Class<? extends OutputFormat<?, ?>> outputFormatClass;
        try {
            outputFormatClass = taskContext.getOutputFormatClass();
        } catch (ClassNotFoundException e) {
            throw new IOException(e);
        }
        ClassLoader outputFormatClassLoader = outputFormatClass.getClassLoader();
        // Use a CombineClassLoader of the output format's classloader and the context classloader
        // This is to prevent class not found issues for classes that are visible to the system, but not to the
        // program/plugin. More specifically, this happens for XML parsers that are in the Hadoop classpath but usually
        // not packaged in programs and plugins.
        // see CDAP-14562 for more info
        ClassLoader outputClassLoader = new CombineClassLoader(outputFormatClassLoader, Thread.currentThread().getContextClassLoader());
        // This is needed in case the OutputFormat's classloader conflicts with the program classloader (for example,
        // TableOutputFormat).
        ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(outputClassLoader);
        try {
            // We use ReflectionUtils to instantiate the OutputFormat, because it also calls setConf on the object, if it
            // is a org.apache.hadoop.conf.Configurable.
            OutputFormat<?, ?> outputFormat = ReflectionUtils.newInstance(outputFormatClass, taskContext.getConfiguration());
            writer = new MeteredRecordWriter<>(outputFormat.getRecordWriter(taskContext), context);
        } finally {
            ClassLoaders.setContextClassLoader(oldClassLoader);
        }
        // add the record-writer to the cache
        recordWriters.put(namedOutput, writer);
    }
    return writer;
}

Example 4 with CombineClassLoader

use of io.cdap.cdap.common.lang.CombineClassLoader in project cdap by caskdata.

the class PluginClassLoaders method createFilteredPluginsClassLoader.

/**
 * Returns a {@link ClassLoader} that only allows loading of plugin classes and plugin exported classes.
 * It should only be used in context when a single ClassLoader is needed to load all different kinds of user classes
 * (e.g. in MapReduce/Spark).
 */
public static ClassLoader createFilteredPluginsClassLoader(Map<String, Plugin> plugins, @Nullable PluginInstantiator pluginInstantiator) {
    if (plugins.isEmpty() || pluginInstantiator == null) {
        return new CombineClassLoader(null);
    }
    try {
        // Gather all explicitly used plugin class names. It is needed for external plugin case.
        Multimap<Plugin, String> artifactPluginClasses = getArtifactPluginClasses(plugins);
        List<ClassLoader> pluginClassLoaders = new ArrayList<>();
        for (Plugin plugin : plugins.values()) {
            ClassLoader pluginClassLoader = pluginInstantiator.getPluginClassLoader(plugin);
            if (pluginClassLoader instanceof PluginClassLoader) {
                // A ClassLoader to allow loading of all plugin classes used by the program.
                Collection<String> pluginClasses = artifactPluginClasses.get(plugin);
                if (!pluginClasses.isEmpty()) {
                    pluginClassLoaders.add(createClassFilteredClassLoader(pluginClasses, pluginClassLoader));
                }
                // A ClassLoader to allow all export package classes to be loadable.
                pluginClassLoaders.add(((PluginClassLoader) pluginClassLoader).getExportPackagesClassLoader());
            }
        }
        return new CombineClassLoader(null, pluginClassLoaders);
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
}

Example 5 with CombineClassLoader

use of io.cdap.cdap.common.lang.CombineClassLoader in project cdap by caskdata.

the class PluginClassLoader method createParent.

static ClassLoader createParent(ClassLoader templateClassLoader) {
    // Find the ProgramClassLoader from the template ClassLoader
    ClassLoader programClassLoader = templateClassLoader;
    while (programClassLoader != null && !(programClassLoader instanceof ProgramClassLoader)) {
        programClassLoader = programClassLoader.getParent();
    }
    // This shouldn't happen
    Preconditions.checkArgument(programClassLoader != null, "Cannot find ProgramClassLoader");
    // Package filtered classloader of the template classloader, which only classes in "Export-Packages" are loadable.
    Manifest manifest = ((ProgramClassLoader) programClassLoader).getManifest();
    Set<String> exportPackages = ManifestFields.getExportPackages(manifest);
    ClassLoader filteredTemplateClassLoader = new PackageFilterClassLoader(templateClassLoader, exportPackages::contains);
    // followed by template export-packages, then by a plugin lib jars.
    return new CombineClassLoader(programClassLoader.getParent(), filteredTemplateClassLoader);
}