Examples with ClassLoaderFolder io.cdap.cdap.common.lang.jar.ClassLoaderFolder used on opensource projects

Example 1 with ClassLoaderFolder

use of io.cdap.cdap.common.lang.jar.ClassLoaderFolder in project cdap by caskdata.

the class AbstractProgramTwillRunnable method doInitialize.

/**
 * Prepares this instance to execute a program.
 *
 * @param programOptionFile a json file containing the serialized {@link ProgramOptions}
 * @throws Exception if failed to initialize
 */
private void doInitialize(File programOptionFile) throws Exception {
    controllerFuture = new CompletableFuture<>();
    programCompletion = new CompletableFuture<>();
    // Setup process wide settings
    Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
    System.setSecurityManager(new ProgramContainerSecurityManager(System.getSecurityManager()));
    SLF4JBridgeHandler.removeHandlersForRootLogger();
    SLF4JBridgeHandler.install();
    // Create the ProgramOptions
    programOptions = createProgramOptions(programOptionFile);
    programRunId = programOptions.getProgramId().run(ProgramRunners.getRunId(programOptions));
    Arguments systemArgs = programOptions.getArguments();
    LoggingContextAccessor.setLoggingContext(LoggingContextHelper.getLoggingContextWithRunId(programRunId, systemArgs.asMap()));
    ClusterMode clusterMode = ProgramRunners.getClusterMode(programOptions);
    // Loads configurations
    Configuration hConf = new Configuration();
    if (clusterMode == ClusterMode.ON_PREMISE) {
        hConf.clear();
        hConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.HADOOP_CONF_FILE)).toURI().toURL());
    }
    UserGroupInformation.setConfiguration(hConf);
    CConfiguration cConf = CConfiguration.create();
    cConf.clear();
    cConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.CDAP_CONF_FILE)).toURI().toURL());
    maxStopSeconds = cConf.getLong(io.cdap.cdap.common.conf.Constants.AppFabric.PROGRAM_MAX_STOP_SECONDS);
    Injector injector = Guice.createInjector(createModule(cConf, hConf, programOptions, programRunId));
    // Initialize log appender
    logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
    logAppenderInitializer.initialize();
    SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer);
    // Setup the proxy selector for in active monitoring mode
    oldProxySelector = ProxySelector.getDefault();
    if (clusterMode == ClusterMode.ISOLATED) {
        RuntimeMonitors.setupMonitoring(injector, programOptions);
    }
    // Create list of core services. They'll will be started in the run method and shutdown when the run
    // method completed
    coreServices = createCoreServices(injector, programOptions);
    // Create the ProgramRunner
    programRunner = createProgramRunner(injector);
    // Create the Program instance
    Location programJarLocation = Locations.toLocation(new File(systemArgs.getOption(ProgramOptionConstants.PROGRAM_JAR)));
    ApplicationSpecification appSpec = readJsonFile(new File(systemArgs.getOption(ProgramOptionConstants.APP_SPEC_FILE)), ApplicationSpecification.class);
    // Expand the program jar for creating classloader
    ClassLoaderFolder classLoaderFolder = BundleJarUtil.prepareClassLoaderFolder(programJarLocation, () -> new File("expanded." + System.currentTimeMillis() + programJarLocation.getName()));
    program = Programs.create(cConf, programRunner, new ProgramDescriptor(programOptions.getProgramId(), appSpec), programJarLocation, classLoaderFolder.getDir());
}

Example 2 with ClassLoaderFolder

use of io.cdap.cdap.common.lang.jar.ClassLoaderFolder in project cdap by caskdata.

the class DefaultRuntimeJob method createProgram.

private Program createProgram(CConfiguration cConf, ProgramRunner programRunner, ProgramDescriptor programDescriptor, ProgramOptions options) throws IOException {
    Location programJarLocation = Locations.toLocation(new File(options.getArguments().getOption(ProgramOptionConstants.PROGRAM_JAR)));
    ClassLoaderFolder classLoaderFolder = BundleJarUtil.prepareClassLoaderFolder(programJarLocation, () -> createTempDirectory(cConf, options.getProgramId(), options.getArguments().getOption(ProgramOptionConstants.RUN_ID)));
    return Programs.create(cConf, programRunner, programDescriptor, programJarLocation, classLoaderFolder.getDir());
}

Example 3 with ClassLoaderFolder

use of io.cdap.cdap.common.lang.jar.ClassLoaderFolder in project cdap by caskdata.

the class ArtifactClassLoaderFactory method createClassLoader.

/**
 * Unpack the given {@code artifactLocation} to a temporary directory and call
 * {@link #createClassLoader(File)} to create the {@link ClassLoader}.
 *
 * @param artifactLocation the location of the artifact to create the classloader from
 * @return a closeable classloader based off the specified artifact; on closing the returned {@link ClassLoader},
 *         all temporary resources created for the classloader will be removed
 * @see #createClassLoader(File)
 */
CloseableClassLoader createClassLoader(Location artifactLocation, EntityImpersonator entityImpersonator) {
    try {
        ClassLoaderFolder classLoaderFolder = entityImpersonator.impersonate(() -> BundleJarUtil.prepareClassLoaderFolder(artifactLocation, () -> DirUtils.createTempDir(tmpDir)));
        CloseableClassLoader classLoader = createClassLoader(classLoaderFolder.getDir());
        return new CloseableClassLoader(classLoader, () -> {
            Closeables.closeQuietly(classLoader);
            Closeables.closeQuietly(classLoaderFolder);
        });
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 4 with ClassLoaderFolder

use of io.cdap.cdap.common.lang.jar.ClassLoaderFolder in project cdap by caskdata.

the class AbstractProgramRuntimeService method createProgram.

/**
 * Creates a {@link Program} for the given {@link ProgramRunner} from the given program jar {@link Location}.
 */
protected Program createProgram(CConfiguration cConf, ProgramRunner programRunner, ProgramDescriptor programDescriptor, ArtifactDetail artifactDetail, final File tempDir) throws IOException {
    Location programJarLocation = artifactDetail.getDescriptor().getLocation();
    ClassLoaderFolder classLoaderFolder;
    try {
        // If the program jar is not a directory, take a snapshot of the jar file to avoid mutation.
        if (!programJarLocation.isDirectory()) {
            File targetFile = new File(tempDir, "program.jar");
            try {
                programJarLocation = Locations.toLocation(Locations.linkOrCopyOverwrite(programJarLocation, targetFile));
            } catch (FileAlreadyExistsException ex) {
                LOG.warn("Program file {} already exists and can not be replaced.", targetFile.getAbsolutePath());
            }
        }
        // Unpack the JAR file
        classLoaderFolder = BundleJarUtil.prepareClassLoaderFolder(programJarLocation, () -> Files.createTempDirectory(tempDir.toPath(), "unpacked").toFile());
    } catch (IOException ioe) {
        throw ioe;
    } catch (Exception e) {
        // should not happen
        throw Throwables.propagate(e);
    }
    return Programs.create(cConf, programRunner, programDescriptor, programJarLocation, classLoaderFolder.getDir());
}

Example 5 with ClassLoaderFolder

use of io.cdap.cdap.common.lang.jar.ClassLoaderFolder in project cdap by caskdata.

the class DatasetTypeManager method addModule.

/**
 * Add datasets module in a namespace
 *
 * @param datasetModuleId the {@link DatasetModuleId} to add
 * @param className module class
 * @param jarLocation location of the module jar
 * @param force if true, an update will be allowed even if there are conflicts with other modules, or if
 *                     removal of a type would break other modules' dependencies.
 */
public void addModule(final DatasetModuleId datasetModuleId, final String className, final Location jarLocation, final boolean force) throws DatasetModuleConflictException {
    LOG.debug("adding module: {}, className: {}, jarLocation: {}", datasetModuleId, className, jarLocation == null ? "[local]" : jarLocation);
    try {
        TransactionRunners.run(transactionRunner, context -> {
            final DatasetTypeTable datasetTypeTable = DatasetTypeTable.create(context);
            final DatasetInstanceTable datasetInstanceTable = new DatasetInstanceTable(context);
            // 1. get existing module with all its types
            DatasetModuleMeta existing = datasetTypeTable.getModule(datasetModuleId);
            DependencyTrackingRegistry reg;
            // 2. unpack jar and create class loader
            ClassLoaderFolder classLoaderFolder = null;
            DirectoryClassLoader cl = null;
            try {
                // NOTE: if jarLocation is null, we assume that this is a system module, ie. always present in classpath
                if (jarLocation != null) {
                    classLoaderFolder = BundleJarUtil.prepareClassLoaderFolder(jarLocation, () -> Files.createTempDirectory(Files.createDirectories(systemTempPath), datasetModuleId.getEntityName()).toFile());
                    cl = new DirectoryClassLoader(classLoaderFolder.getDir(), cConf.get(Constants.AppFabric.PROGRAM_EXTRA_CLASSPATH), FilterClassLoader.create(getClass().getClassLoader()), "lib");
                }
                reg = new DependencyTrackingRegistry(datasetModuleId, datasetTypeTable, cl, force);
                // 3. register the new module while tracking dependencies.
                // this will fail if a type exists in a different module
                DatasetDefinitionRegistries.register(className, cl, reg);
            } catch (TypeConflictException e) {
                // type conflict from the registry, we want to throw that as is
                throw e;
            } catch (Exception e) {
                LOG.error("Could not instantiate instance of dataset module class {} for module {} using jarLocation {}", className, datasetModuleId, jarLocation);
                throw Throwables.propagate(e);
            } finally {
                // Close the ProgramClassLoader
                Closeables.closeQuietly(cl);
                Closeables.closeQuietly(classLoaderFolder);
            }
            // 4. determine whether any type were removed from the module, and whether any other modules depend on them
            if (existing != null) {
                Set<String> removedTypes = new HashSet<>(existing.getTypes());
                removedTypes.removeAll(reg.getTypes());
                // TODO (CDAP-6294): track dependencies at the type level
                if (!force && !removedTypes.isEmpty() && !existing.getUsedByModules().isEmpty()) {
                    throw new DatasetModuleConflictException(String.format("Cannot update module '%s' to remove types %s: Modules %s may depend on it. Delete them first", datasetModuleId, removedTypes, existing.getUsedByModules()));
                }
                Collection<DatasetSpecification> instances = datasetInstanceTable.getByTypes(datasetModuleId.getParent(), removedTypes);
                if (!instances.isEmpty()) {
                    throw new DatasetModuleConflictException(String.format("Attempt to remove dataset types %s from module '%s' that have existing instances: %s. " + "Delete them first.", removedTypes, datasetModuleId, instances.stream().map(input -> input.getName() + ":" + input.getType()).collect(Collectors.joining(", "))));
                }
            }
            // NOTE: we use set to avoid duplicated dependencies
            // NOTE: we use LinkedHashSet to preserve order in which dependencies must be loaded
            Set<String> moduleDependencies = new LinkedHashSet<String>();
            for (DatasetTypeId usedType : reg.getUsedTypes()) {
                DatasetModuleMeta usedModule = datasetTypeTable.getModuleByType(usedType);
                if (usedModule == null) {
                    throw new IllegalStateException(String.format("Found a null used module for type %s for while adding module %s", usedType, datasetModuleId));
                }
                // adding all used types and the module itself, in this very order to keep the order of loading modules
                // for instantiating a type
                moduleDependencies.addAll(usedModule.getUsesModules());
                boolean added = moduleDependencies.add(usedModule.getName());
                if (added) {
                    // also adding this module as a dependent for all modules it uses
                    usedModule.addUsedByModule(datasetModuleId.getEntityName());
                    datasetTypeTable.writeModule(usedType.getParent(), usedModule);
                }
            }
            URI jarURI = jarLocation == null ? null : jarLocation.toURI();
            DatasetModuleMeta moduleMeta = existing == null ? new DatasetModuleMeta(datasetModuleId.getEntityName(), className, jarURI, reg.getTypes(), Lists.newArrayList(moduleDependencies)) : new DatasetModuleMeta(datasetModuleId.getEntityName(), className, jarURI, reg.getTypes(), Lists.newArrayList(moduleDependencies), Lists.newArrayList(existing.getUsedByModules()));
            datasetTypeTable.writeModule(datasetModuleId.getParent(), moduleMeta);
        });
    } catch (RuntimeException e) {
        for (Throwable cause : Throwables.getCausalChain(e)) {
            if (cause instanceof DatasetModuleConflictException) {
                throw (DatasetModuleConflictException) cause;
            } else if (cause instanceof TypeConflictException) {
                throw new DatasetModuleConflictException(cause.getMessage(), cause);
            }
        }
        throw Throwables.propagate(e);
    } catch (Exception e) {
        LOG.error("Operation failed", e);
        throw Throwables.propagate(e);
    }
}