Examples with SparkRunnerClassLoader co.cask.cdap.app.runtime.spark.classloader.SparkRunnerClassLoader used on opensource projects

Example 1 with SparkRunnerClassLoader

use of co.cask.cdap.app.runtime.spark.classloader.SparkRunnerClassLoader in project cdap by caskdata.

the class SparkProgramRuntimeProvider method createClassLoader.

/**
 * Returns an array of {@link URL} being used by the {@link ClassLoader} of this {@link Class}.
 */
private synchronized SparkRunnerClassLoader createClassLoader(boolean filterScalaClasses, boolean rewriteYarnClient) throws IOException {
    // Determine if needs to filter Scala classes or not.
    ClassLoader runnerParentClassLoader = filterScalaClasses ? new ScalaFilterClassLoader(getClass().getClassLoader()) : getClass().getClassLoader();
    if (classLoaderUrls == null) {
        classLoaderUrls = getSparkClassloaderURLs(getClass().getClassLoader());
    }
    SparkRunnerClassLoader runnerClassLoader = new SparkRunnerClassLoader(classLoaderUrls, runnerParentClassLoader, rewriteYarnClient);
    try {
        // Use reflection to call scala.reflect.runtime.package$.MODULE$.universe.runtimeMirror(classLoader)
        // The scala.reflect.runtime.package$ is the class, which has a public MODULE$ field, which is scala way of
        // doing singleton object.
        // We use reflection to avoid code dependency  on the scala version.
        Object scalaReflect = runnerClassLoader.loadClass("scala.reflect.runtime.package$").getField("MODULE$").get(null);
        Object javaMirrors = scalaReflect.getClass().getMethod("universe").invoke(scalaReflect);
        javaMirrors.getClass().getMethod("runtimeMirror", ClassLoader.class).invoke(javaMirrors, runnerClassLoader);
    } catch (ClassNotFoundException | NoSuchMethodException | InvocationTargetException e) {
        // The SI-6240 is fixed in Scala 2.11 anyway and older Scala version should have the classes and methods
        // that we try to invoke above.
        LOG.debug("Not able to create scala runtime mirror for SparkRunnerClassLoader. " + "This can happen if there is incompatible Scala API changes with Scala version newer than 2.10. " + "However, the SI-6240 bug is fixed since 2.11, hence the workaround for the the bug is not needed ");
    } catch (Exception e) {
        LOG.warn("Failed to create scala runtime mirror for SparkRunnerClassLoader. " + "Running multiple Spark from the same JVM process might fail due to Scala reflection bug SI-6240.", e);
    }
    return runnerClassLoader;
}