use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.
the class AbstractProgramRuntimeServiceTest method testDeadlock.
@Test(timeout = 5000)
public void testDeadlock() throws IOException, ExecutionException, InterruptedException, TimeoutException {
// This test is for testing condition in (CDAP-3579)
// The race condition is if a program finished very fast such that inside the AbstractProgramRuntimeService is
// still in the run method, it holds the object lock, making the callback from the listener block forever.
ProgramRunnerFactory runnerFactory = createProgramRunnerFactory();
Program program = createDummyProgram();
ProgramRuntimeService runtimeService = new TestProgramRuntimeService(CConfiguration.create(), runnerFactory, program, null, null);
runtimeService.startAndWait();
try {
ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, NamespaceId.DEFAULT.artifact("test", "1.0"));
ProgramController controller = runtimeService.run(descriptor, new SimpleProgramOptions(program.getId()), RunIds.generate()).getController();
Tasks.waitFor(ProgramController.State.COMPLETED, controller::getState, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
Tasks.waitFor(true, () -> runtimeService.list(ProgramType.WORKER).isEmpty(), 5, TimeUnit.SECONDS, 100, TimeUnit.MICROSECONDS);
} finally {
runtimeService.stopAndWait();
}
}
use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.
the class AbstractProgramRuntimeServiceTest method testConcurrentStartLimit.
@Test
public void testConcurrentStartLimit() throws Exception {
Semaphore proceed = new Semaphore(0);
Set<String> threadNames = ConcurrentHashMap.newKeySet();
// Creates a runner factory that block start until a signal is received
ProgramRunnerFactory runnerFactory = programType -> (program, options) -> {
threadNames.add(Thread.currentThread().getName());
proceed.acquireUninterruptibly();
ProgramId programId = program.getId();
Service service = new FastService();
ProgramController controller = new ProgramControllerServiceAdapter(service, programId.run(RunIds.generate()));
service.start();
return controller;
};
Program program = createDummyProgram();
ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, NamespaceId.DEFAULT.artifact("test", "1.0"));
CConfiguration cConf = CConfiguration.create();
cConf.setInt(Constants.AppFabric.PROGRAM_LAUNCH_THREADS, 2);
ProgramRuntimeService runtimeService = new TestProgramRuntimeService(cConf, runnerFactory, program, null, null);
runtimeService.startAndWait();
try {
List<ProgramController> controllers = new ArrayList<>();
for (int i = 0; i < 5; i++) {
controllers.add(runtimeService.run(descriptor, new SimpleProgramOptions(program.getId()), RunIds.generate()).getController());
}
// There should be 2 program start threads running only
Tasks.waitFor(2, proceed::getQueueLength, 2, TimeUnit.SECONDS);
try {
// Shouldn't never have more than 2
Tasks.waitFor(true, () -> proceed.getQueueLength() > 2, 2, TimeUnit.SECONDS);
Assert.fail("Shouldn't have more than 2 program starting");
} catch (TimeoutException e) {
// expected
}
// Let all start to proceed. They should have all finished pretty quickly
proceed.release(5);
Tasks.waitFor(true, () -> controllers.stream().map(ProgramController::getState).allMatch(ProgramController.State.COMPLETED::equals), 5, TimeUnit.SECONDS);
Assert.assertEquals(2, threadNames.size());
} finally {
runtimeService.stopAndWait();
}
}
use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.
the class RuntimeServiceMainTest method testRuntimeService.
@Test
public void testRuntimeService() throws Exception {
ArtifactId artifactId = NamespaceId.DEFAULT.artifact("test", "1.0");
ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").worker("worker").run(RunIds.generate());
Map<String, String> systemArgs = ImmutableMap.of(SystemArguments.PROFILE_PROVISIONER, NativeProvisioner.SPEC.getName(), SystemArguments.PROFILE_NAME, "default");
ProgramOptions programOptions = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments());
ProgramDescriptor programDescriptor = new ProgramDescriptor(programRunId.getParent(), null, artifactId);
// Write out program state events to simulate program start
Injector appFabricInjector = getServiceMainInstance(AppFabricServiceMain.class).getInjector();
CConfiguration cConf = appFabricInjector.getInstance(CConfiguration.class);
ProgramStatePublisher programStatePublisher = new MessagingProgramStatePublisher(appFabricInjector.getInstance(MessagingService.class), NamespaceId.SYSTEM.topic(cConf.get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC)), RetryStrategies.fromConfiguration(cConf, "system.program.state."));
new MessagingProgramStateWriter(programStatePublisher).start(programRunId, programOptions, null, programDescriptor);
Injector injector = getServiceMainInstance(RuntimeServiceMain.class).getInjector();
TransactionRunner txRunner = injector.getInstance(TransactionRunner.class);
// Should see a STARTING record in the runtime store
Tasks.waitFor(ProgramRunStatus.STARTING, () -> {
RunRecordDetail detail = TransactionRunners.run(txRunner, context -> {
return AppMetadataStore.create(context).getRun(programRunId);
});
return detail == null ? null : detail.getStatus();
}, 5, TimeUnit.SECONDS);
ProgramStateWriter programStateWriter = createProgramStateWriter(injector, programRunId);
// Write a running state. We should see a RUNNING record in the runtime store
programStateWriter.running(programRunId, null);
Tasks.waitFor(ProgramRunStatus.RUNNING, () -> {
RunRecordDetail detail = TransactionRunners.run(txRunner, context -> {
return AppMetadataStore.create(context).getRun(programRunId);
});
return detail == null ? null : detail.getStatus();
}, 5, TimeUnit.SECONDS);
// Write a complete state. The run record should be removed in the runtime store
programStateWriter.completed(programRunId);
Tasks.waitFor(true, () -> TransactionRunners.run(txRunner, context -> AppMetadataStore.create(context).getRun(programRunId) == null), 5, TimeUnit.SECONDS);
}
use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.
the class DistributedWorkflowProgramRunnerTest method setupWorkflowRuntime.
/**
* Setup the {@link ProgramLaunchConfig} for the given workflow.
*/
private ProgramLaunchConfig setupWorkflowRuntime(String workflowName, Map<String, String> runtimeArgs) throws IOException {
// Create the distributed workflow program runner
ProgramRunner programRunner = programRunnerFactory.create(ProgramType.WORKFLOW);
Assert.assertTrue(programRunner instanceof DistributedWorkflowProgramRunner);
DistributedWorkflowProgramRunner workflowRunner = (DistributedWorkflowProgramRunner) programRunner;
// Create the Workflow Program
Program workflowProgram = createWorkflowProgram(cConf, programRunner, workflowName);
ProgramLaunchConfig launchConfig = new ProgramLaunchConfig();
ProgramOptions programOpts = new SimpleProgramOptions(workflowProgram.getId(), new BasicArguments(), new BasicArguments(runtimeArgs));
// Setup the launching config
workflowRunner.setupLaunchConfig(launchConfig, workflowProgram, programOpts, cConf, new Configuration(), TEMP_FOLDER.newFolder());
return launchConfig;
}
use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.
the class AbstractProgramRuntimeService method createPluginSnapshot.
/**
* Return the copy of the {@link ProgramOptions} including locations of plugin artifacts in it.
* @param options the {@link ProgramOptions} in which the locations of plugin artifacts needs to be included
* @param programId Id of the Program
* @param tempDir Temporary Directory to create the plugin artifact snapshot
* @param appSpec program's Application Specification
* @return the copy of the program options with locations of plugin artifacts included in them
*/
private ProgramOptions createPluginSnapshot(ProgramOptions options, ProgramId programId, File tempDir, @Nullable ApplicationSpecification appSpec) throws Exception {
// appSpec is null in an unit test
if (appSpec == null) {
return options;
}
Set<String> files = Sets.newHashSet();
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
builder.putAll(options.getArguments().asMap());
for (Map.Entry<String, Plugin> pluginEntry : appSpec.getPlugins().entrySet()) {
Plugin plugin = pluginEntry.getValue();
File destFile = new File(tempDir, Artifacts.getFileName(plugin.getArtifactId()));
// Skip if the file has already been copied.
if (!files.add(destFile.getName())) {
continue;
}
try {
ArtifactId artifactId = Artifacts.toProtoArtifactId(programId.getNamespaceId(), plugin.getArtifactId());
copyArtifact(artifactId, noAuthArtifactRepository.getArtifact(Id.Artifact.fromEntityId(artifactId)), destFile);
} catch (ArtifactNotFoundException e) {
throw new IllegalArgumentException(String.format("Artifact %s could not be found", plugin.getArtifactId()), e);
}
}
LOG.debug("Plugin artifacts of {} copied to {}", programId, tempDir.getAbsolutePath());
builder.put(ProgramOptionConstants.PLUGIN_DIR, tempDir.getAbsolutePath());
return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(builder.build()), options.getUserArguments(), options.isDebug());
}
Aggregations