use of io.cdap.cdap.internal.app.program.MessagingProgramStateWriter in project cdap by caskdata.
the class RuntimeClientServiceTest method testRuntimeClientStop.
/**
* Test for {@link RuntimeClientService} that will block termination until a program completed mess
*/
@Test(timeout = 10000L)
public void testRuntimeClientStop() throws Exception {
ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
ListenableFuture<Service.State> stopFuture = runtimeClientService.stop();
try {
stopFuture.get(2, TimeUnit.SECONDS);
Assert.fail("Expected runtime client service not stopped");
} catch (TimeoutException e) {
// Expected
}
// Publish a program completed state, which should unblock the client service stop.
programStateWriter.completed(PROGRAM_RUN_ID);
stopFuture.get();
}
use of io.cdap.cdap.internal.app.program.MessagingProgramStateWriter in project cdap by caskdata.
the class RuntimeClientServiceTest method testBasicRelay.
@Test
public void testBasicRelay() throws Exception {
// Send some messages to multiple topics in the client side TMS, they should get replicated to the server side TMS.
MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
// the RuntimeClientService will decode it to watch for program termination
if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
// Write a non-terminal state to test basic relaying
programStateWriter.running(PROGRAM_RUN_ID, null);
} else {
messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), entry.getKey(), entry.getKey());
}
}
MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
// Extract the program run status from the Notification
Tasks.waitFor(Collections.singletonList(ProgramRunStatus.RUNNING), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
} else {
Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
}
}
// Writes a program terminate message to unblock stopping of the client service
programStateWriter.completed(PROGRAM_RUN_ID);
}
use of io.cdap.cdap.internal.app.program.MessagingProgramStateWriter in project cdap by caskdata.
the class RuntimeClientServiceTest method testProgramTerminate.
/**
* Test for {@link RuntimeClientService} that will terminate itself when seeing program completed message.
*/
@Test
public void testProgramTerminate() throws Exception {
MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
// Send a terminate program state first, wait for the service sees the state change,
// then publish messages to other topics.
programStateWriter.completed(PROGRAM_RUN_ID);
Tasks.waitFor(true, () -> runtimeClientService.getProgramFinishTime() >= 0, 2, TimeUnit.SECONDS);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
// the RuntimeClientService will decode it to watch for program termination
if (!entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
List<String> payloads = Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey());
messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), StandardCharsets.UTF_8, payloads.iterator());
}
}
// The client service should get stopped by itself.
Tasks.waitFor(Service.State.TERMINATED, () -> runtimeClientService.state(), clientCConf.getLong(Constants.RuntimeMonitor.GRACEFUL_SHUTDOWN_MS) + 2000, TimeUnit.MILLISECONDS);
// All messages should be sent after the runtime client service stopped
MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
// Extract the program run status from the Notification
Tasks.waitFor(Collections.singletonList(ProgramRunStatus.COMPLETED), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
} else {
Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
}
}
}
use of io.cdap.cdap.internal.app.program.MessagingProgramStateWriter in project cdap by caskdata.
the class CoreSchedulerServiceTest method testProgramEvents.
@Test
@Category(XSlowTests.class)
public void testProgramEvents() throws Exception {
// Deploy the app
deploy(AppWithMultipleSchedules.class, 200);
CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
TopicId programEventTopic = NamespaceId.SYSTEM.topic(cConf.get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC));
ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(cConf, messagingService);
// These notifications should not trigger the program
ProgramRunId anotherWorkflowRun = ANOTHER_WORKFLOW.run(RunIds.generate());
ArtifactId artifactId = ANOTHER_WORKFLOW.getNamespaceId().artifact("test", "1.0").toApiArtifactId();
ApplicationSpecification appSpec = new DefaultApplicationSpecification(AppWithMultipleSchedules.NAME, ApplicationId.DEFAULT_VERSION, ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
ProgramDescriptor programDescriptor = new ProgramDescriptor(anotherWorkflowRun.getParent(), appSpec);
BasicArguments systemArgs = new BasicArguments(ImmutableMap.of(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString()));
ProgramOptions programOptions = new SimpleProgramOptions(anotherWorkflowRun.getParent(), systemArgs, new BasicArguments(), false);
programStateWriter.start(anotherWorkflowRun, programOptions, null, programDescriptor);
programStateWriter.running(anotherWorkflowRun, null);
long lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
programStateWriter.error(anotherWorkflowRun, null);
waitUntilProcessed(programEventTopic, lastProcessed);
ProgramRunId someWorkflowRun = SOME_WORKFLOW.run(RunIds.generate());
programDescriptor = new ProgramDescriptor(someWorkflowRun.getParent(), appSpec);
programStateWriter.start(someWorkflowRun, new SimpleProgramOptions(someWorkflowRun.getParent(), systemArgs, new BasicArguments()), null, programDescriptor);
programStateWriter.running(someWorkflowRun, null);
lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
programStateWriter.killed(someWorkflowRun);
waitUntilProcessed(programEventTopic, lastProcessed);
Assert.assertEquals(0, getRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.ALL));
// Enable the schedule
scheduler.enableSchedule(APP_MULT_ID.schedule(AppWithMultipleSchedules.WORKFLOW_COMPLETED_SCHEDULE));
// Start a program with user arguments
startProgram(ANOTHER_WORKFLOW, ImmutableMap.of(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_KEY, AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE), 200);
// Wait for a completed run record
waitForCompleteRuns(1, TRIGGERED_WORKFLOW);
assertProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED, 1);
RunRecord run = getProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED).get(0);
Map<String, List<WorkflowTokenDetail.NodeValueDetail>> tokenData = getWorkflowToken(TRIGGERED_WORKFLOW, run.getPid(), null, null).getTokenData();
// There should be 2 entries in tokenData
Assert.assertEquals(2, tokenData.size());
// The value of TRIGGERED_RUNTIME_ARG_KEY should be ANOTHER_RUNTIME_ARG_VALUE from the triggering workflow
Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_RUNTIME_ARG_KEY).get(0).getValue());
// The value of TRIGGERED_TOKEN_KEY should be ANOTHER_TOKEN_VALUE from the triggering workflow
Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_TOKEN_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_TOKEN_KEY).get(0).getValue());
}
use of io.cdap.cdap.internal.app.program.MessagingProgramStateWriter in project cdap by caskdata.
the class RuntimeServiceMainTest method testRuntimeService.
@Test
public void testRuntimeService() throws Exception {
ArtifactId artifactId = NamespaceId.DEFAULT.artifact("test", "1.0");
ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").worker("worker").run(RunIds.generate());
Map<String, String> systemArgs = ImmutableMap.of(SystemArguments.PROFILE_PROVISIONER, NativeProvisioner.SPEC.getName(), SystemArguments.PROFILE_NAME, "default");
ProgramOptions programOptions = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments());
ProgramDescriptor programDescriptor = new ProgramDescriptor(programRunId.getParent(), null, artifactId);
// Write out program state events to simulate program start
Injector appFabricInjector = getServiceMainInstance(AppFabricServiceMain.class).getInjector();
CConfiguration cConf = appFabricInjector.getInstance(CConfiguration.class);
ProgramStatePublisher programStatePublisher = new MessagingProgramStatePublisher(appFabricInjector.getInstance(MessagingService.class), NamespaceId.SYSTEM.topic(cConf.get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC)), RetryStrategies.fromConfiguration(cConf, "system.program.state."));
new MessagingProgramStateWriter(programStatePublisher).start(programRunId, programOptions, null, programDescriptor);
Injector injector = getServiceMainInstance(RuntimeServiceMain.class).getInjector();
TransactionRunner txRunner = injector.getInstance(TransactionRunner.class);
// Should see a STARTING record in the runtime store
Tasks.waitFor(ProgramRunStatus.STARTING, () -> {
RunRecordDetail detail = TransactionRunners.run(txRunner, context -> {
return AppMetadataStore.create(context).getRun(programRunId);
});
return detail == null ? null : detail.getStatus();
}, 5, TimeUnit.SECONDS);
ProgramStateWriter programStateWriter = createProgramStateWriter(injector, programRunId);
// Write a running state. We should see a RUNNING record in the runtime store
programStateWriter.running(programRunId, null);
Tasks.waitFor(ProgramRunStatus.RUNNING, () -> {
RunRecordDetail detail = TransactionRunners.run(txRunner, context -> {
return AppMetadataStore.create(context).getRun(programRunId);
});
return detail == null ? null : detail.getStatus();
}, 5, TimeUnit.SECONDS);
// Write a complete state. The run record should be removed in the runtime store
programStateWriter.completed(programRunId);
Tasks.waitFor(true, () -> TransactionRunners.run(txRunner, context -> AppMetadataStore.create(context).getRun(programRunId) == null), 5, TimeUnit.SECONDS);
}
Aggregations