Search in sources :

Example 1 with ApplicationSpecification

use of io.cdap.cdap.api.app.ApplicationSpecification in project cdap by caskdata.

the class DistributedMapReduceProgramRunner method setupLaunchConfig.

@Override
protected void setupLaunchConfig(ProgramLaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) {
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    MapReduceSpecification spec = appSpec.getMapReduce().get(program.getName());
    // Get the resource for the container that runs the mapred client that will launch the actual mapred job.
    Map<String, String> clientArgs = RuntimeArguments.extractScope("task", "client", options.getUserArguments().asMap());
    // Add runnable. Only one instance for the MR driver
    launchConfig.addRunnable(spec.getName(), new MapReduceTwillRunnable(spec.getName()), 1, clientArgs, spec.getDriverResources(), 0);
    if (clusterMode == ClusterMode.ON_PREMISE || cConf.getBoolean(Constants.AppFabric.PROGRAM_REMOTE_RUNNER, false)) {
        // Add extra resources, classpath and dependencies
        launchConfig.addExtraResources(MapReduceContainerHelper.localizeFramework(hConf, new HashMap<>())).addExtraClasspath(MapReduceContainerHelper.addMapReduceClassPath(hConf, new ArrayList<>())).addExtraDependencies(YarnClientProtocolProvider.class);
    }
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) HashMap(java.util.HashMap) MapReduceSpecification(io.cdap.cdap.api.mapreduce.MapReduceSpecification)

Example 2 with ApplicationSpecification

use of io.cdap.cdap.api.app.ApplicationSpecification in project cdap by caskdata.

the class DistributedMapReduceProgramRunner method validateOptions.

@Override
protected void validateOptions(Program program, ProgramOptions options) {
    super.validateOptions(program, options);
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.MAPREDUCE, "Only MapReduce process type is supported.");
    MapReduceSpecification spec = appSpec.getMapReduce().get(program.getName());
    Preconditions.checkNotNull(spec, "Missing MapReduceSpecification for %s", program.getName());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) MapReduceSpecification(io.cdap.cdap.api.mapreduce.MapReduceSpecification) ProgramType(io.cdap.cdap.proto.ProgramType)

Example 3 with ApplicationSpecification

use of io.cdap.cdap.api.app.ApplicationSpecification in project cdap by caskdata.

the class AbstractProgramTwillRunnable method doInitialize.

/**
 * Prepares this instance to execute a program.
 *
 * @param programOptionFile a json file containing the serialized {@link ProgramOptions}
 * @throws Exception if failed to initialize
 */
private void doInitialize(File programOptionFile) throws Exception {
    controllerFuture = new CompletableFuture<>();
    programCompletion = new CompletableFuture<>();
    // Setup process wide settings
    Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
    System.setSecurityManager(new ProgramContainerSecurityManager(System.getSecurityManager()));
    SLF4JBridgeHandler.removeHandlersForRootLogger();
    SLF4JBridgeHandler.install();
    // Create the ProgramOptions
    programOptions = createProgramOptions(programOptionFile);
    programRunId = programOptions.getProgramId().run(ProgramRunners.getRunId(programOptions));
    Arguments systemArgs = programOptions.getArguments();
    LoggingContextAccessor.setLoggingContext(LoggingContextHelper.getLoggingContextWithRunId(programRunId, systemArgs.asMap()));
    ClusterMode clusterMode = ProgramRunners.getClusterMode(programOptions);
    // Loads configurations
    Configuration hConf = new Configuration();
    if (clusterMode == ClusterMode.ON_PREMISE) {
        hConf.clear();
        hConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.HADOOP_CONF_FILE)).toURI().toURL());
    }
    UserGroupInformation.setConfiguration(hConf);
    CConfiguration cConf = CConfiguration.create();
    cConf.clear();
    cConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.CDAP_CONF_FILE)).toURI().toURL());
    maxStopSeconds = cConf.getLong(io.cdap.cdap.common.conf.Constants.AppFabric.PROGRAM_MAX_STOP_SECONDS);
    Injector injector = Guice.createInjector(createModule(cConf, hConf, programOptions, programRunId));
    // Initialize log appender
    logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
    logAppenderInitializer.initialize();
    SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer);
    // Setup the proxy selector for in active monitoring mode
    oldProxySelector = ProxySelector.getDefault();
    if (clusterMode == ClusterMode.ISOLATED) {
        RuntimeMonitors.setupMonitoring(injector, programOptions);
    }
    // Create list of core services. They'll will be started in the run method and shutdown when the run
    // method completed
    coreServices = createCoreServices(injector, programOptions);
    // Create the ProgramRunner
    programRunner = createProgramRunner(injector);
    // Create the Program instance
    Location programJarLocation = Locations.toLocation(new File(systemArgs.getOption(ProgramOptionConstants.PROGRAM_JAR)));
    ApplicationSpecification appSpec = readJsonFile(new File(systemArgs.getOption(ProgramOptionConstants.APP_SPEC_FILE)), ApplicationSpecification.class);
    // Expand the program jar for creating classloader
    ClassLoaderFolder classLoaderFolder = BundleJarUtil.prepareClassLoaderFolder(programJarLocation, () -> new File("expanded." + System.currentTimeMillis() + programJarLocation.getName()));
    program = Programs.create(cConf, programRunner, new ProgramDescriptor(programOptions.getProgramId(), appSpec), programJarLocation, classLoaderFolder.getDir());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) Configuration(org.apache.hadoop.conf.Configuration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ClusterMode(io.cdap.cdap.app.guice.ClusterMode) Arguments(io.cdap.cdap.app.runtime.Arguments) SystemArguments(io.cdap.cdap.internal.app.runtime.SystemArguments) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ClassLoaderFolder(io.cdap.cdap.common.lang.jar.ClassLoaderFolder) LogAppenderInitializer(io.cdap.cdap.logging.appender.LogAppenderInitializer) Injector(com.google.inject.Injector) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) UncaughtExceptionHandler(io.cdap.cdap.common.logging.common.UncaughtExceptionHandler) File(java.io.File) Location(org.apache.twill.filesystem.Location)

Example 4 with ApplicationSpecification

use of io.cdap.cdap.api.app.ApplicationSpecification in project cdap by caskdata.

the class DistributedServiceProgramRunner method setupLaunchConfig.

@Override
protected void setupLaunchConfig(ProgramLaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) {
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    ServiceSpecification serviceSpec = appSpec.getServices().get(program.getName());
    // Add a runnable for the service handler
    launchConfig.addRunnable(serviceSpec.getName(), new ServiceTwillRunnable(serviceSpec.getName()), serviceSpec.getInstances(), options.getUserArguments().asMap(), serviceSpec.getResources());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) ServiceSpecification(io.cdap.cdap.api.service.ServiceSpecification)

Example 5 with ApplicationSpecification

use of io.cdap.cdap.api.app.ApplicationSpecification in project cdap by caskdata.

the class DistributedServiceProgramRunner method validateOptions.

@Override
protected void validateOptions(Program program, ProgramOptions options) {
    super.validateOptions(program, options);
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.SERVICE, "Only SERVICE process type is supported.");
    ServiceSpecification serviceSpec = appSpec.getServices().get(program.getName());
    Preconditions.checkNotNull(serviceSpec, "Missing ServiceSpecification for %s", program.getName());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) ServiceSpecification(io.cdap.cdap.api.service.ServiceSpecification) ProgramType(io.cdap.cdap.proto.ProgramType)

Aggregations

ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)114 ApplicationId (io.cdap.cdap.proto.id.ApplicationId)57 ProgramId (io.cdap.cdap.proto.id.ProgramId)45 Test (org.junit.Test)45 AllProgramsApp (io.cdap.cdap.AllProgramsApp)27 ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)26 ProgramType (io.cdap.cdap.proto.ProgramType)25 ArrayList (java.util.ArrayList)21 HashMap (java.util.HashMap)20 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)19 ArtifactId (io.cdap.cdap.api.artifact.ArtifactId)16 NotFoundException (io.cdap.cdap.common.NotFoundException)15 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)12 IOException (java.io.IOException)12 Map (java.util.Map)12 ProgramDescriptor (io.cdap.cdap.app.program.ProgramDescriptor)11 HashSet (java.util.HashSet)11 ApplicationNotFoundException (io.cdap.cdap.common.ApplicationNotFoundException)10 Id (io.cdap.cdap.common.id.Id)9 ProgramSpecification (io.cdap.cdap.api.ProgramSpecification)8