Examples with CombineClassLoader co.cask.cdap.common.lang.CombineClassLoader used on opensource projects

Example 1 with CombineClassLoader

use of co.cask.cdap.common.lang.CombineClassLoader in project cdap by caskdata.

the class AbstractHttpHandlerDelegator method wrapResponder.

/**
   * Returns a new instance of {@link DelayedHttpServiceResponder} that wraps around the given {@link HttpResponder}
   * object. This method is called from handler class generated by {@link HttpHandlerGenerator}.
   */
@SuppressWarnings("unused")
protected final DelayedHttpServiceResponder wrapResponder(HttpResponder responder) {
    MetricsContext collector = this.metricsContext;
    HttpServiceContext serviceContext = context.getServiceContext();
    Preconditions.checkState(serviceContext instanceof TransactionalHttpServiceContext, "This instance of HttpServiceContext does not support transactions.");
    if (serviceContext.getSpecification() != null) {
        collector = metricsContext.childContext(Constants.Metrics.Tag.HANDLER, serviceContext.getSpecification().getName());
    }
    return new DelayedHttpServiceResponder(responder, new BodyProducerFactory() {

        @Override
        public BodyProducer create(HttpContentProducer contentProducer, TransactionalHttpServiceContext serviceContext) {
            final ClassLoader programContextClassLoader = new CombineClassLoader(null, ImmutableList.of(contentProducer.getClass().getClassLoader(), getClass().getClassLoader()));
            // Capture the context since we need to keep it until the end of the content producing.
            // We don't need to worry about double capturing of the context when HttpContentConsumer is used.
            // This is because when HttpContentConsumer is used, the responder constructed here will get closed and this
            // BodyProducerFactory won't be used.
            final Cancellable contextReleaser = context.capture();
            return new BodyProducerAdapter(contentProducer, serviceContext, programContextClassLoader, contextReleaser);
        }
    }, (TransactionalHttpServiceContext) serviceContext, collector);
}

Example 2 with CombineClassLoader

use of co.cask.cdap.common.lang.CombineClassLoader in project cdap by caskdata.

the class AbstractHttpHandlerDelegator method wrapContentConsumer.

/**
   * Returns a new instance of {@link BodyConsumer} that wraps around the given {@link HttpContentConsumer}
   * and {@link DelayedHttpServiceResponder}.
   *
   * IMPORTANT: This method will also capture the context associated with the current thread, hence after
   * this method is called, no other methods on this class should be called from the current thread.
   *
   * This method is called from handler class generated by {@link HttpHandlerGenerator}.
   */
@SuppressWarnings("unused")
protected final BodyConsumer wrapContentConsumer(HttpContentConsumer consumer, DelayedHttpServiceResponder responder) {
    Preconditions.checkState(!responder.hasBufferedResponse(), "HttpContentConsumer may not be used after a response has already been sent.");
    // Close the provided responder since a new one will be created for the BodyConsumerAdapter to use.
    responder.close();
    final HttpServiceContext serviceContext = context.getServiceContext();
    Preconditions.checkState(serviceContext instanceof TransactionalHttpServiceContext, "This instance of HttpServiceContext does not support transactions.");
    final Cancellable contextReleaser = context.capture();
    final ClassLoader programContextClassLoader = new CombineClassLoader(null, ImmutableList.of(consumer.getClass().getClassLoader(), getClass().getClassLoader()));
    return new BodyConsumerAdapter(new DelayedHttpServiceResponder(responder, new BodyProducerFactory() {

        @Override
        public BodyProducer create(HttpContentProducer contentProducer, TransactionalHttpServiceContext serviceContext) {
            // Transfer the captured context from the content consumer to the content producer
            return new BodyProducerAdapter(contentProducer, serviceContext, programContextClassLoader, contextReleaser);
        }
    }), consumer, serviceContext, programContextClassLoader, contextReleaser);
}

Example 3 with CombineClassLoader

use of co.cask.cdap.common.lang.CombineClassLoader in project cdap by caskdata.

the class MainClassLoader method createFromContext.

/**
 * @param filter A {@link FilterClassLoader.Filter} for filtering out classes from the
 * @param extraClasspath extra list of {@link URL} to be added to the end of the classpath for the
 *                       {@link MainClassLoader} to be created
 * @return a new instance from the current context classloader or the system classloader. The returned
 * {@link MainClassLoader} will be the defining classloader for classes in the context classloader
 * that the filter rejected. For classes that pass the filter, the defining classloader will be the original
 * context classloader.
 * It will return {@code null} if it is not able to create a new instance due to lack of classpath information.
 */
@Nullable
public static MainClassLoader createFromContext(FilterClassLoader.Filter filter, URL... extraClasspath) {
    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
    if (classLoader == null) {
        classLoader = ClassLoader.getSystemClassLoader();
    }
    List<URL> classpath = new ArrayList<>();
    if (classLoader instanceof URLClassLoader) {
        classpath.addAll(Arrays.asList(((URLClassLoader) classLoader).getURLs()));
    } else if (classLoader == ClassLoader.getSystemClassLoader()) {
        addClassPath(classpath);
    } else {
        // No able to create a new MainClassLoader
        return null;
    }
    classpath.addAll(Arrays.asList(extraClasspath));
    ClassLoader filtered = new FilterClassLoader(classLoader, filter);
    ClassLoader parent = new CombineClassLoader(classLoader.getParent(), filtered);
    return new MainClassLoader(classpath.toArray(new URL[classpath.size()]), parent);
}

Example 4 with CombineClassLoader

use of co.cask.cdap.common.lang.CombineClassLoader in project cdap by caskdata.

the class InMemoryConfigurator method getSpecJson.

private <T extends Config> String getSpecJson(Application<T> app) throws Exception {
    // This Gson cannot be static since it is used to deserialize user class.
    // Gson will keep a static map to class, hence will leak the classloader
    Gson gson = new GsonBuilder().registerTypeAdapterFactory(new CaseInsensitiveEnumTypeAdapterFactory()).create();
    // Now, we call configure, which returns application specification.
    DefaultAppConfigurer configurer;
    File tempDir = DirUtils.createTempDir(baseUnpackDir);
    try (PluginInstantiator pluginInstantiator = new PluginInstantiator(cConf, app.getClass().getClassLoader(), tempDir)) {
        configurer = new DefaultAppConfigurer(appNamespace, artifactId, app, configString, artifactRepository, pluginInstantiator);
        T appConfig;
        Type configType = Artifacts.getConfigType(app.getClass());
        if (configString.isEmpty()) {
            // noinspection unchecked
            appConfig = ((Class<T>) configType).newInstance();
        } else {
            try {
                appConfig = gson.fromJson(configString, configType);
            } catch (JsonSyntaxException e) {
                throw new IllegalArgumentException("Invalid JSON configuration was provided. Please check the syntax.", e);
            }
        }
        try {
            ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(new CombineClassLoader(null, app.getClass().getClassLoader(), getClass().getClassLoader()));
            try {
                app.configure(configurer, new DefaultApplicationContext<>(appConfig));
            } finally {
                ClassLoaders.setContextClassLoader(oldClassLoader);
            }
        } catch (Throwable t) {
            Throwable rootCause = Throwables.getRootCause(t);
            if (rootCause instanceof ClassNotFoundException) {
                // Heuristic to provide better error message
                String missingClass = rootCause.getMessage();
                // If the missing class has "spark" in the name, try to see if Spark is available
                if (missingClass.startsWith("org.apache.spark.") || missingClass.startsWith("co.cask.cdap.api.spark.")) {
                    // Try to load the SparkContext class, which should be available if Spark is available in the platform
                    try {
                        artifactClassLoader.loadClass("org.apache.spark.SparkContext");
                    } catch (ClassNotFoundException e) {
                        // Spark is not available, it is most likely caused by missing Spark in the platform
                        throw new IllegalStateException("Missing Spark related class " + missingClass + ". It may be caused by unavailability of Spark. " + "Please verify environment variable " + Constants.SPARK_HOME + " is set correctly", t);
                    }
                    // Spark is available, can be caused by incompatible Spark version
                    throw new InvalidArtifactException("Missing Spark related class " + missingClass + ". Configured to use Spark located at " + System.getenv(Constants.SPARK_HOME) + ", which may be incompatible with the one required by the application", t);
                }
                // then the missing class is most likely due to some missing library in the artifact jar
                throw new InvalidArtifactException("Missing class " + missingClass + ". It may be caused by missing dependency jar(s) in the artifact jar.", t);
            }
            throw t;
        }
    } finally {
        try {
            DirUtils.deleteDirectoryContents(tempDir);
        } catch (IOException e) {
            LOG.warn("Exception raised when deleting directory {}", tempDir, e);
        }
    }
    ApplicationSpecification specification = configurer.createSpecification(applicationName, applicationVersion);
    // TODO: The SchemaGenerator should be injected
    return ApplicationSpecificationAdapter.create(new ReflectionSchemaGenerator()).toJson(specification);
}

Example 5 with CombineClassLoader

use of co.cask.cdap.common.lang.CombineClassLoader in project cdap by caskdata.

the class DistributedProgramRunner method run.

@Override
public final ProgramController run(final Program program, ProgramOptions oldOptions) {
    validateOptions(program, oldOptions);
    final CConfiguration cConf = createContainerCConf(this.cConf);
    final Configuration hConf = createContainerHConf(this.hConf);
    final File tempDir = DirUtils.createTempDir(new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile());
    try {
        final LaunchConfig launchConfig = new LaunchConfig();
        setupLaunchConfig(launchConfig, program, oldOptions, cConf, hConf, tempDir);
        // Add extra localize resources needed by the program runner
        final Map<String, LocalizeResource> localizeResources = new HashMap<>(launchConfig.getExtraResources());
        final ProgramOptions options = addArtifactPluginFiles(oldOptions, localizeResources, DirUtils.createTempDir(tempDir));
        final List<String> additionalClassPaths = new ArrayList<>();
        addContainerJars(cConf, localizeResources, additionalClassPaths);
        prepareHBaseDDLExecutorResources(tempDir, cConf, localizeResources);
        // Save the configuration to files
        final File hConfFile = saveHConf(hConf, new File(tempDir, HADOOP_CONF_FILE_NAME));
        final File cConfFile = saveCConf(cConf, new File(tempDir, CDAP_CONF_FILE_NAME));
        // Localize the program jar
        Location programJarLocation = program.getJarLocation();
        final String programJarName = programJarLocation.getName();
        localizeResources.put(programJarName, new LocalizeResource(program.getJarLocation().toURI(), false));
        // Localize an expanded program jar
        final String expandedProgramJarName = "expanded." + programJarName;
        localizeResources.put(expandedProgramJarName, new LocalizeResource(program.getJarLocation().toURI(), true));
        // Localize the app spec
        localizeResources.put(APP_SPEC_FILE_NAME, new LocalizeResource(saveAppSpec(program, File.createTempFile("appSpec", ".json", tempDir))));
        final URI logbackURI = getLogBackURI(program, tempDir);
        if (logbackURI != null) {
            // Localize the logback xml
            localizeResources.put(LOGBACK_FILE_NAME, new LocalizeResource(logbackURI, false));
        }
        Callable<ProgramController> callable = new Callable<ProgramController>() {

            @Override
            public ProgramController call() throws Exception {
                ProgramTwillApplication twillApplication = new ProgramTwillApplication(program.getId(), launchConfig.getRunnables(), launchConfig.getLaunchOrder(), localizeResources, createEventHandler(cConf, options));
                TwillPreparer twillPreparer = twillRunner.prepare(twillApplication);
                // Add the configuration to container classpath
                twillPreparer.withResources(hConfFile.toURI(), cConfFile.toURI());
                Map<String, String> userArgs = options.getUserArguments().asMap();
                // Setup log level
                twillPreparer.setLogLevels(transformLogLevels(SystemArguments.getLogLevels(userArgs)));
                // Set the configuration for the twill application
                Map<String, String> twillConfigs = new HashMap<>();
                if (DistributedProgramRunner.this instanceof LongRunningDistributedProgramRunner) {
                    twillConfigs.put(Configs.Keys.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL, cConf.get(Constants.AppFabric.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL));
                }
                // Add the one from the runtime arguments
                twillConfigs.putAll(SystemArguments.getTwillApplicationConfigs(userArgs));
                twillPreparer.withConfiguration(twillConfigs);
                // Setup per runnable configurations
                for (Map.Entry<String, RunnableDefinition> entry : launchConfig.getRunnables().entrySet()) {
                    String runnable = entry.getKey();
                    RunnableDefinition runnableDefinition = entry.getValue();
                    if (runnableDefinition.getMaxRetries() != null) {
                        twillPreparer.withMaxRetries(runnable, runnableDefinition.getMaxRetries());
                    }
                    twillPreparer.setLogLevels(runnable, transformLogLevels(runnableDefinition.getLogLevels()));
                    twillPreparer.withConfiguration(runnable, runnableDefinition.getTwillRunnableConfigs());
                }
                if (options.isDebug()) {
                    twillPreparer.enableDebugging();
                }
                logProgramStart(program, options);
                String serializedOptions = GSON.toJson(options, ProgramOptions.class);
                LOG.info("Starting {} with debugging enabled: {}, programOptions: {}, and logback: {}", program.getId(), options.isDebug(), serializedOptions, logbackURI);
                // Add scheduler queue name if defined
                String schedulerQueueName = options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE);
                if (schedulerQueueName != null && !schedulerQueueName.isEmpty()) {
                    LOG.info("Setting scheduler queue for app {} as {}", program.getId(), schedulerQueueName);
                    twillPreparer.setSchedulerQueue(schedulerQueueName);
                }
                if (logbackURI != null) {
                    twillPreparer.addJVMOptions("-Dlogback.configurationFile=" + LOGBACK_FILE_NAME);
                }
                String logLevelConf = cConf.get(Constants.COLLECT_APP_CONTAINER_LOG_LEVEL).toUpperCase();
                if ("OFF".equals(logLevelConf)) {
                    twillPreparer.withConfiguration(Collections.singletonMap(Configs.Keys.LOG_COLLECTION_ENABLED, "false"));
                } else {
                    LogEntry.Level logLevel = LogEntry.Level.ERROR;
                    if ("ALL".equals(logLevelConf)) {
                        logLevel = LogEntry.Level.TRACE;
                    } else {
                        try {
                            logLevel = LogEntry.Level.valueOf(logLevelConf.toUpperCase());
                        } catch (Exception e) {
                            LOG.warn("Invalid application container log level {}. Defaulting to ERROR.", logLevelConf);
                        }
                    }
                    twillPreparer.addLogHandler(new LoggerLogHandler(LOG, logLevel));
                }
                // Add secure tokens
                if (User.isHBaseSecurityEnabled(hConf) || UserGroupInformation.isSecurityEnabled()) {
                    twillPreparer.addSecureStore(YarnSecureStore.create(secureStoreRenewer.createCredentials()));
                }
                // Setup the environment for the container logback.xml
                twillPreparer.withEnv(Collections.singletonMap("CDAP_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR));
                // Add dependencies
                Set<Class<?>> extraDependencies = new HashSet<>(launchConfig.getExtraDependencies());
                extraDependencies.add(HBaseTableUtilFactory.getHBaseTableUtilClass(cConf));
                if (SecureStoreUtils.isKMSBacked(cConf) && SecureStoreUtils.isKMSCapable()) {
                    extraDependencies.add(SecureStoreUtils.getKMSSecureStore());
                }
                twillPreparer.withDependencies(extraDependencies);
                // Add the additional classes to the classpath that comes from the container jar setting
                twillPreparer.withClassPaths(additionalClassPaths);
                twillPreparer.withClassPaths(launchConfig.getExtraClasspath());
                twillPreparer.withEnv(launchConfig.getExtraEnv());
                // The Yarn app classpath goes last
                List<String> yarnAppClassPath = Arrays.asList(hConf.getTrimmedStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
                twillPreparer.withApplicationClassPaths(yarnAppClassPath).withClassPaths(yarnAppClassPath).withBundlerClassAcceptor(launchConfig.getClassAcceptor()).withApplicationArguments("--" + RunnableOptions.JAR, programJarName, "--" + RunnableOptions.EXPANDED_JAR, expandedProgramJarName, "--" + RunnableOptions.HADOOP_CONF_FILE, HADOOP_CONF_FILE_NAME, "--" + RunnableOptions.CDAP_CONF_FILE, CDAP_CONF_FILE_NAME, "--" + RunnableOptions.APP_SPEC_FILE, APP_SPEC_FILE_NAME, "--" + RunnableOptions.PROGRAM_OPTIONS, serializedOptions, "--" + RunnableOptions.PROGRAM_ID, GSON.toJson(program.getId())).setClassLoader(MainClassLoader.class.getName());
                // Invoke the before launch hook
                beforeLaunch(program, options);
                TwillController twillController;
                // Change the context classloader to the combine classloader of this ProgramRunner and
                // all the classloaders of the dependencies classes so that Twill can trace classes.
                ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(new CombineClassLoader(DistributedProgramRunner.this.getClass().getClassLoader(), Iterables.transform(extraDependencies, new Function<Class<?>, ClassLoader>() {

                    @Override
                    public ClassLoader apply(Class<?> input) {
                        return input.getClassLoader();
                    }
                })));
                try {
                    twillController = twillPreparer.start(cConf.getLong(Constants.AppFabric.PROGRAM_MAX_START_SECONDS), TimeUnit.SECONDS);
                } finally {
                    ClassLoaders.setContextClassLoader(oldClassLoader);
                }
                return createProgramController(addCleanupListener(twillController, program, tempDir), new ProgramDescriptor(program.getId(), program.getApplicationSpecification()), ProgramRunners.getRunId(options));
            }
        };
        return impersonator.doAs(program.getId(), callable);
    } catch (Exception e) {
        deleteDirectory(tempDir);
        throw Throwables.propagate(e);
    }
}