use of io.cdap.cdap.app.runtime.ProgramRunner in project cdap by caskdata.
the class DefaultRuntimeJob method run.
@Override
public void run(RuntimeJobEnvironment runtimeJobEnv) throws Exception {
// Setup process wide settings
Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
SLF4JBridgeHandler.removeHandlersForRootLogger();
SLF4JBridgeHandler.install();
// Get Program Options
ProgramOptions programOpts = readJsonFile(new File(DistributedProgramRunner.PROGRAM_OPTIONS_FILE_NAME), ProgramOptions.class);
ProgramRunId programRunId = programOpts.getProgramId().run(ProgramRunners.getRunId(programOpts));
ProgramId programId = programRunId.getParent();
Arguments systemArgs = programOpts.getArguments();
// Setup logging context for the program
LoggingContextAccessor.setLoggingContext(LoggingContextHelper.getLoggingContextWithRunId(programRunId, systemArgs.asMap()));
// Get the cluster launch type
Cluster cluster = GSON.fromJson(systemArgs.getOption(ProgramOptionConstants.CLUSTER), Cluster.class);
// Get App spec
ApplicationSpecification appSpec = readJsonFile(new File(DistributedProgramRunner.APP_SPEC_FILE_NAME), ApplicationSpecification.class);
ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec);
// Create injector and get program runner
Injector injector = Guice.createInjector(createModules(runtimeJobEnv, createCConf(runtimeJobEnv, programOpts), programRunId, programOpts));
CConfiguration cConf = injector.getInstance(CConfiguration.class);
// Initialize log appender
LogAppenderInitializer logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
logAppenderInitializer.initialize();
SystemArguments.setLogLevel(programOpts.getUserArguments(), logAppenderInitializer);
ProxySelector oldProxySelector = ProxySelector.getDefault();
RuntimeMonitors.setupMonitoring(injector, programOpts);
Deque<Service> coreServices = createCoreServices(injector, systemArgs, cluster);
startCoreServices(coreServices);
// regenerate app spec
ConfiguratorFactory configuratorFactory = injector.getInstance(ConfiguratorFactory.class);
try {
Map<String, String> systemArguments = new HashMap<>(programOpts.getArguments().asMap());
File pluginDir = new File(programOpts.getArguments().getOption(ProgramOptionConstants.PLUGIN_DIR, DistributedProgramRunner.PLUGIN_DIR));
// create a directory to store plugin artifacts for the regeneration of app spec to fetch plugin artifacts
DirUtils.mkdirs(pluginDir);
if (!programOpts.getArguments().hasOption(ProgramOptionConstants.PLUGIN_DIR)) {
systemArguments.put(ProgramOptionConstants.PLUGIN_DIR, DistributedProgramRunner.PLUGIN_DIR);
}
// remember the file names in the artifact folder before app regeneration
List<String> pluginFiles = DirUtils.listFiles(pluginDir, File::isFile).stream().map(File::getName).collect(Collectors.toList());
ApplicationSpecification generatedAppSpec = regenerateAppSpec(systemArguments, programOpts.getUserArguments().asMap(), programId, appSpec, programDescriptor, configuratorFactory);
appSpec = generatedAppSpec != null ? generatedAppSpec : appSpec;
programDescriptor = new ProgramDescriptor(programDescriptor.getProgramId(), appSpec);
List<String> pluginFilesAfter = DirUtils.listFiles(pluginDir, File::isFile).stream().map(File::getName).collect(Collectors.toList());
if (pluginFilesAfter.isEmpty()) {
systemArguments.remove(ProgramOptionConstants.PLUGIN_DIR);
}
// recreate it from the folders
if (!pluginFiles.equals(pluginFilesAfter)) {
systemArguments.remove(ProgramOptionConstants.PLUGIN_ARCHIVE);
}
// update program options
programOpts = new SimpleProgramOptions(programOpts.getProgramId(), new BasicArguments(systemArguments), programOpts.getUserArguments(), programOpts.isDebug());
} catch (Exception e) {
LOG.warn("Failed to regenerate the app spec for program {}, using the existing app spec", programId, e);
}
ProgramStateWriter programStateWriter = injector.getInstance(ProgramStateWriter.class);
RuntimeClientService runtimeClientService = injector.getInstance(RuntimeClientService.class);
CompletableFuture<ProgramController.State> programCompletion = new CompletableFuture<>();
try {
ProgramRunner programRunner = injector.getInstance(ProgramRunnerFactory.class).create(programId.getType());
// Create and run the program. The program files should be present in current working directory.
try (Program program = createProgram(cConf, programRunner, programDescriptor, programOpts)) {
ProgramController controller = programRunner.run(program, programOpts);
controllerFuture.complete(controller);
runtimeClientService.onProgramStopRequested(controller::stop);
controller.addListener(new AbstractListener() {
@Override
public void completed() {
programCompletion.complete(ProgramController.State.COMPLETED);
}
@Override
public void killed() {
// Write an extra state to make sure there is always a terminal state even
// if the program application run failed to write out the state.
programStateWriter.killed(programRunId);
programCompletion.complete(ProgramController.State.KILLED);
}
@Override
public void error(Throwable cause) {
// Write an extra state to make sure there is always a terminal state even
// if the program application run failed to write out the state.
programStateWriter.error(programRunId, cause);
programCompletion.completeExceptionally(cause);
}
}, Threads.SAME_THREAD_EXECUTOR);
if (stopRequested) {
controller.stop();
}
// Block on the completion
programCompletion.get();
} finally {
if (programRunner instanceof Closeable) {
Closeables.closeQuietly((Closeable) programRunner);
}
}
} catch (Throwable t) {
controllerFuture.completeExceptionally(t);
if (!programCompletion.isDone()) {
// We log here so that the logs would still send back to the program logs collection.
// Only log if the program completion is not done.
// Otherwise the program runner itself should have logged the error.
LOG.error("Failed to execute program {}", programRunId, t);
// If the program completion is not done, then this exception
// is due to systematic failure in which fail to run the program.
// We write out an extra error state for the program to make sure the program state get transited.
programStateWriter.error(programRunId, t);
}
throw t;
} finally {
stopCoreServices(coreServices, logAppenderInitializer);
ProxySelector.setDefault(oldProxySelector);
Authenticator.setDefault(null);
runCompletedLatch.countDown();
}
}
use of io.cdap.cdap.app.runtime.ProgramRunner in project cdap by caskdata.
the class DistributedWorkflowProgramRunner method setupLaunchConfig.
@Override
protected void setupLaunchConfig(ProgramLaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) throws IOException {
WorkflowSpecification spec = program.getApplicationSpecification().getWorkflows().get(program.getName());
List<ClassAcceptor> acceptors = new ArrayList<>();
acceptors.add(launchConfig.getClassAcceptor());
// Only interested in MapReduce and Spark nodes.
// This is because CUSTOM_ACTION types are running inside the driver
Set<SchedulableProgramType> runnerTypes = EnumSet.of(SchedulableProgramType.MAPREDUCE, SchedulableProgramType.SPARK);
Iterable<ScheduleProgramInfo> programInfos = spec.getNodeIdMap().values().stream().filter(WorkflowActionNode.class::isInstance).map(WorkflowActionNode.class::cast).map(WorkflowActionNode::getProgram).filter(programInfo -> runnerTypes.contains(programInfo.getProgramType()))::iterator;
// Can't use Stream.forEach as we want to preserve the IOException being thrown
for (ScheduleProgramInfo programInfo : programInfos) {
ProgramType programType = ProgramType.valueOfSchedulableType(programInfo.getProgramType());
ProgramRunner runner = programRunnerFactory.create(programType);
try {
if (runner instanceof DistributedProgramRunner) {
// Call setupLaunchConfig with the corresponding program.
// Need to constructs a new ProgramOptions with the scope extracted for the given program
ProgramId programId = program.getId().getParent().program(programType, programInfo.getProgramName());
Map<String, String> programUserArgs = RuntimeArguments.extractScope(programId.getType().getScope(), programId.getProgram(), options.getUserArguments().asMap());
ProgramOptions programOptions = new SimpleProgramOptions(programId, options.getArguments(), new BasicArguments(programUserArgs));
((DistributedProgramRunner) runner).setupLaunchConfig(launchConfig, Programs.create(cConf, program, programId, runner), programOptions, cConf, hConf, tempDir);
acceptors.add(launchConfig.getClassAcceptor());
}
} finally {
if (runner instanceof Closeable) {
Closeables.closeQuietly((Closeable) runner);
}
}
}
// Set the class acceptor
launchConfig.setClassAcceptor(new AndClassAcceptor(acceptors));
// Find out the default resources requirements based on the programs inside the workflow
// At least gives the Workflow driver 768 mb of container memory
Map<String, Resources> runnablesResources = Maps.transformValues(launchConfig.getRunnables(), this::getResources);
Resources defaultResources = maxResources(new Resources(768), findDriverResources(spec.getNodes(), runnablesResources));
// Clear and set the runnable for the workflow driver.
launchConfig.clearRunnables();
// Extract scoped runtime arguments that only meant for the workflow but not for child nodes
Map<String, String> runtimeArgs = RuntimeArguments.extractScope("task", "workflow", options.getUserArguments().asMap());
launchConfig.addRunnable(spec.getName(), new WorkflowTwillRunnable(spec.getName()), 1, runtimeArgs, defaultResources, 0);
}
use of io.cdap.cdap.app.runtime.ProgramRunner in project cdap by caskdata.
the class InMemoryProgramRunnerModule method configure.
/**
* Configures a {@link com.google.inject.Binder} via the exposed methods.
*/
@Override
protected void configure() {
// Bind ServiceAnnouncer for service.
bind(ServiceAnnouncer.class).to(DiscoveryServiceAnnouncer.class);
// Bind the ArtifactManager implementation and expose it.
// It could used by ProgramRunner loaded through runtime extension.
install(new FactoryModuleBuilder().implement(ArtifactManager.class, LocalArtifactManager.class).build(ArtifactManagerFactory.class));
expose(ArtifactManagerFactory.class);
// Bind ProgramRunner
MapBinder<ProgramType, ProgramRunner> runnerFactoryBinder = MapBinder.newMapBinder(binder(), ProgramType.class, ProgramRunner.class);
// Programs with multiple instances have an InMemoryProgramRunner that starts threads to manage all of their
// instances.
runnerFactoryBinder.addBinding(ProgramType.MAPREDUCE).to(MapReduceProgramRunner.class);
runnerFactoryBinder.addBinding(ProgramType.WORKFLOW).to(WorkflowProgramRunner.class);
runnerFactoryBinder.addBinding(ProgramType.WORKER).to(InMemoryWorkerRunner.class);
runnerFactoryBinder.addBinding(ProgramType.SERVICE).to(InMemoryServiceProgramRunner.class);
// Bind program runners in private scope
// They should only be used by the ProgramRunners in the runnerFactoryBinder
bind(ServiceProgramRunner.class);
bind(WorkerProgramRunner.class);
// ProgramRunnerFactory should be in local mode
bind(ProgramRuntimeProvider.Mode.class).toInstance(ProgramRuntimeProvider.Mode.LOCAL);
bind(ProgramRunnerFactory.class).to(DefaultProgramRunnerFactory.class).in(Scopes.SINGLETON);
// Note: Expose for test cases. Need to refactor test cases.
expose(ProgramRunnerFactory.class);
// Bind and expose runtime service
bind(ProgramRuntimeService.class).to(InMemoryProgramRuntimeService.class).in(Scopes.SINGLETON);
expose(ProgramRuntimeService.class);
}
use of io.cdap.cdap.app.runtime.ProgramRunner in project cdap by caskdata.
the class DefaultProgramRunnerFactory method create.
@Override
public ProgramRunner create(ProgramType programType) {
ProgramRuntimeProvider provider = runtimeProviderLoader.get(programType);
ProgramRunner runner;
if (provider != null) {
LOG.trace("Using runtime provider {} for program type {}", provider, programType);
runner = provider.createProgramRunner(programType, mode, injector);
} else {
Provider<ProgramRunner> defaultProvider = defaultRunnerProviders.get(programType);
if (defaultProvider == null) {
throw new IllegalArgumentException("Unsupported program type: " + programType);
}
runner = defaultProvider.get();
}
// In distributed mode, program state changes are recorded in the event handler by Twill AM.
return (mode == ProgramRuntimeProvider.Mode.LOCAL || publishProgramState) ? wrapProgramRunner(runner) : runner;
}
use of io.cdap.cdap.app.runtime.ProgramRunner in project cdap by cdapio.
the class DefaultRuntimeJob method run.
@Override
public void run(RuntimeJobEnvironment runtimeJobEnv) throws Exception {
// Setup process wide settings
Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
SLF4JBridgeHandler.removeHandlersForRootLogger();
SLF4JBridgeHandler.install();
// Get Program Options
ProgramOptions programOpts = readJsonFile(new File(DistributedProgramRunner.PROGRAM_OPTIONS_FILE_NAME), ProgramOptions.class);
ProgramRunId programRunId = programOpts.getProgramId().run(ProgramRunners.getRunId(programOpts));
ProgramId programId = programRunId.getParent();
Arguments systemArgs = programOpts.getArguments();
// Setup logging context for the program
LoggingContextAccessor.setLoggingContext(LoggingContextHelper.getLoggingContextWithRunId(programRunId, systemArgs.asMap()));
// Get the cluster launch type
Cluster cluster = GSON.fromJson(systemArgs.getOption(ProgramOptionConstants.CLUSTER), Cluster.class);
// Get App spec
ApplicationSpecification appSpec = readJsonFile(new File(DistributedProgramRunner.APP_SPEC_FILE_NAME), ApplicationSpecification.class);
ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec);
// Create injector and get program runner
Injector injector = Guice.createInjector(createModules(runtimeJobEnv, createCConf(runtimeJobEnv, programOpts), programRunId, programOpts));
CConfiguration cConf = injector.getInstance(CConfiguration.class);
// Initialize log appender
LogAppenderInitializer logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
logAppenderInitializer.initialize();
SystemArguments.setLogLevel(programOpts.getUserArguments(), logAppenderInitializer);
ProxySelector oldProxySelector = ProxySelector.getDefault();
RuntimeMonitors.setupMonitoring(injector, programOpts);
Deque<Service> coreServices = createCoreServices(injector, systemArgs, cluster);
startCoreServices(coreServices);
// regenerate app spec
ConfiguratorFactory configuratorFactory = injector.getInstance(ConfiguratorFactory.class);
try {
Map<String, String> systemArguments = new HashMap<>(programOpts.getArguments().asMap());
File pluginDir = new File(programOpts.getArguments().getOption(ProgramOptionConstants.PLUGIN_DIR, DistributedProgramRunner.PLUGIN_DIR));
// create a directory to store plugin artifacts for the regeneration of app spec to fetch plugin artifacts
DirUtils.mkdirs(pluginDir);
if (!programOpts.getArguments().hasOption(ProgramOptionConstants.PLUGIN_DIR)) {
systemArguments.put(ProgramOptionConstants.PLUGIN_DIR, DistributedProgramRunner.PLUGIN_DIR);
}
// remember the file names in the artifact folder before app regeneration
List<String> pluginFiles = DirUtils.listFiles(pluginDir, File::isFile).stream().map(File::getName).collect(Collectors.toList());
ApplicationSpecification generatedAppSpec = regenerateAppSpec(systemArguments, programOpts.getUserArguments().asMap(), programId, appSpec, programDescriptor, configuratorFactory);
appSpec = generatedAppSpec != null ? generatedAppSpec : appSpec;
programDescriptor = new ProgramDescriptor(programDescriptor.getProgramId(), appSpec);
List<String> pluginFilesAfter = DirUtils.listFiles(pluginDir, File::isFile).stream().map(File::getName).collect(Collectors.toList());
if (pluginFilesAfter.isEmpty()) {
systemArguments.remove(ProgramOptionConstants.PLUGIN_DIR);
}
// recreate it from the folders
if (!pluginFiles.equals(pluginFilesAfter)) {
systemArguments.remove(ProgramOptionConstants.PLUGIN_ARCHIVE);
}
// update program options
programOpts = new SimpleProgramOptions(programOpts.getProgramId(), new BasicArguments(systemArguments), programOpts.getUserArguments(), programOpts.isDebug());
} catch (Exception e) {
LOG.warn("Failed to regenerate the app spec for program {}, using the existing app spec", programId, e);
}
ProgramStateWriter programStateWriter = injector.getInstance(ProgramStateWriter.class);
RuntimeClientService runtimeClientService = injector.getInstance(RuntimeClientService.class);
CompletableFuture<ProgramController.State> programCompletion = new CompletableFuture<>();
try {
ProgramRunner programRunner = injector.getInstance(ProgramRunnerFactory.class).create(programId.getType());
// Create and run the program. The program files should be present in current working directory.
try (Program program = createProgram(cConf, programRunner, programDescriptor, programOpts)) {
ProgramController controller = programRunner.run(program, programOpts);
controllerFuture.complete(controller);
runtimeClientService.onProgramStopRequested(controller::stop);
controller.addListener(new AbstractListener() {
@Override
public void completed() {
programCompletion.complete(ProgramController.State.COMPLETED);
}
@Override
public void killed() {
// Write an extra state to make sure there is always a terminal state even
// if the program application run failed to write out the state.
programStateWriter.killed(programRunId);
programCompletion.complete(ProgramController.State.KILLED);
}
@Override
public void error(Throwable cause) {
// Write an extra state to make sure there is always a terminal state even
// if the program application run failed to write out the state.
programStateWriter.error(programRunId, cause);
programCompletion.completeExceptionally(cause);
}
}, Threads.SAME_THREAD_EXECUTOR);
if (stopRequested) {
controller.stop();
}
// Block on the completion
programCompletion.get();
} finally {
if (programRunner instanceof Closeable) {
Closeables.closeQuietly((Closeable) programRunner);
}
}
} catch (Throwable t) {
controllerFuture.completeExceptionally(t);
if (!programCompletion.isDone()) {
// We log here so that the logs would still send back to the program logs collection.
// Only log if the program completion is not done.
// Otherwise the program runner itself should have logged the error.
LOG.error("Failed to execute program {}", programRunId, t);
// If the program completion is not done, then this exception
// is due to systematic failure in which fail to run the program.
// We write out an extra error state for the program to make sure the program state get transited.
programStateWriter.error(programRunId, t);
}
throw t;
} finally {
stopCoreServices(coreServices, logAppenderInitializer);
ProxySelector.setDefault(oldProxySelector);
Authenticator.setDefault(null);
runCompletedLatch.countDown();
}
}
Aggregations