use of io.cdap.cdap.api.app.ApplicationSpecification in project cdap by caskdata.
the class DistributedMapReduceProgramRunner method setupLaunchConfig.
@Override
protected void setupLaunchConfig(ProgramLaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) {
ApplicationSpecification appSpec = program.getApplicationSpecification();
MapReduceSpecification spec = appSpec.getMapReduce().get(program.getName());
// Get the resource for the container that runs the mapred client that will launch the actual mapred job.
Map<String, String> clientArgs = RuntimeArguments.extractScope("task", "client", options.getUserArguments().asMap());
// Add runnable. Only one instance for the MR driver
launchConfig.addRunnable(spec.getName(), new MapReduceTwillRunnable(spec.getName()), 1, clientArgs, spec.getDriverResources(), 0);
if (clusterMode == ClusterMode.ON_PREMISE || cConf.getBoolean(Constants.AppFabric.PROGRAM_REMOTE_RUNNER, false)) {
// Add extra resources, classpath and dependencies
launchConfig.addExtraResources(MapReduceContainerHelper.localizeFramework(hConf, new HashMap<>())).addExtraClasspath(MapReduceContainerHelper.addMapReduceClassPath(hConf, new ArrayList<>())).addExtraDependencies(YarnClientProtocolProvider.class);
}
}
use of io.cdap.cdap.api.app.ApplicationSpecification in project cdap by caskdata.
the class DistributedMapReduceProgramRunner method validateOptions.
@Override
protected void validateOptions(Program program, ProgramOptions options) {
super.validateOptions(program, options);
// Extract and verify parameters
ApplicationSpecification appSpec = program.getApplicationSpecification();
Preconditions.checkNotNull(appSpec, "Missing application specification.");
ProgramType processorType = program.getType();
Preconditions.checkNotNull(processorType, "Missing processor type.");
Preconditions.checkArgument(processorType == ProgramType.MAPREDUCE, "Only MapReduce process type is supported.");
MapReduceSpecification spec = appSpec.getMapReduce().get(program.getName());
Preconditions.checkNotNull(spec, "Missing MapReduceSpecification for %s", program.getName());
}
use of io.cdap.cdap.api.app.ApplicationSpecification in project cdap by caskdata.
the class AbstractProgramTwillRunnable method doInitialize.
/**
* Prepares this instance to execute a program.
*
* @param programOptionFile a json file containing the serialized {@link ProgramOptions}
* @throws Exception if failed to initialize
*/
private void doInitialize(File programOptionFile) throws Exception {
controllerFuture = new CompletableFuture<>();
programCompletion = new CompletableFuture<>();
// Setup process wide settings
Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
System.setSecurityManager(new ProgramContainerSecurityManager(System.getSecurityManager()));
SLF4JBridgeHandler.removeHandlersForRootLogger();
SLF4JBridgeHandler.install();
// Create the ProgramOptions
programOptions = createProgramOptions(programOptionFile);
programRunId = programOptions.getProgramId().run(ProgramRunners.getRunId(programOptions));
Arguments systemArgs = programOptions.getArguments();
LoggingContextAccessor.setLoggingContext(LoggingContextHelper.getLoggingContextWithRunId(programRunId, systemArgs.asMap()));
ClusterMode clusterMode = ProgramRunners.getClusterMode(programOptions);
// Loads configurations
Configuration hConf = new Configuration();
if (clusterMode == ClusterMode.ON_PREMISE) {
hConf.clear();
hConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.HADOOP_CONF_FILE)).toURI().toURL());
}
UserGroupInformation.setConfiguration(hConf);
CConfiguration cConf = CConfiguration.create();
cConf.clear();
cConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.CDAP_CONF_FILE)).toURI().toURL());
maxStopSeconds = cConf.getLong(io.cdap.cdap.common.conf.Constants.AppFabric.PROGRAM_MAX_STOP_SECONDS);
Injector injector = Guice.createInjector(createModule(cConf, hConf, programOptions, programRunId));
// Initialize log appender
logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
logAppenderInitializer.initialize();
SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer);
// Setup the proxy selector for in active monitoring mode
oldProxySelector = ProxySelector.getDefault();
if (clusterMode == ClusterMode.ISOLATED) {
RuntimeMonitors.setupMonitoring(injector, programOptions);
}
// Create list of core services. They'll will be started in the run method and shutdown when the run
// method completed
coreServices = createCoreServices(injector, programOptions);
// Create the ProgramRunner
programRunner = createProgramRunner(injector);
// Create the Program instance
Location programJarLocation = Locations.toLocation(new File(systemArgs.getOption(ProgramOptionConstants.PROGRAM_JAR)));
ApplicationSpecification appSpec = readJsonFile(new File(systemArgs.getOption(ProgramOptionConstants.APP_SPEC_FILE)), ApplicationSpecification.class);
// Expand the program jar for creating classloader
ClassLoaderFolder classLoaderFolder = BundleJarUtil.prepareClassLoaderFolder(programJarLocation, () -> new File("expanded." + System.currentTimeMillis() + programJarLocation.getName()));
program = Programs.create(cConf, programRunner, new ProgramDescriptor(programOptions.getProgramId(), appSpec), programJarLocation, classLoaderFolder.getDir());
}
use of io.cdap.cdap.api.app.ApplicationSpecification in project cdap by caskdata.
the class DistributedServiceProgramRunner method setupLaunchConfig.
@Override
protected void setupLaunchConfig(ProgramLaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) {
ApplicationSpecification appSpec = program.getApplicationSpecification();
ServiceSpecification serviceSpec = appSpec.getServices().get(program.getName());
// Add a runnable for the service handler
launchConfig.addRunnable(serviceSpec.getName(), new ServiceTwillRunnable(serviceSpec.getName()), serviceSpec.getInstances(), options.getUserArguments().asMap(), serviceSpec.getResources());
}
use of io.cdap.cdap.api.app.ApplicationSpecification in project cdap by caskdata.
the class DistributedServiceProgramRunner method validateOptions.
@Override
protected void validateOptions(Program program, ProgramOptions options) {
super.validateOptions(program, options);
// Extract and verify parameters
ApplicationSpecification appSpec = program.getApplicationSpecification();
Preconditions.checkNotNull(appSpec, "Missing application specification.");
ProgramType processorType = program.getType();
Preconditions.checkNotNull(processorType, "Missing processor type.");
Preconditions.checkArgument(processorType == ProgramType.SERVICE, "Only SERVICE process type is supported.");
ServiceSpecification serviceSpec = appSpec.getServices().get(program.getName());
Preconditions.checkNotNull(serviceSpec, "Missing ServiceSpecification for %s", program.getName());
}
Aggregations