use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class RuntimeClientServerTest method testFutureIsNotBlockingWhenValueIsSet.
@Test
public void testFutureIsNotBlockingWhenValueIsSet() throws Exception {
CountDownLatch countDownLatch = new CountDownLatch(1);
runtimeClient.onProgramStopRequested(() -> countDownLatch.countDown());
// Now call sendMessages which will set the future
ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("workflow").run(RunIds.generate());
TopicId topicId = NamespaceId.SYSTEM.topic("topic");
List<Message> messages = new ArrayList<>();
messages.add(createMessage(Math.max(1, RuntimeClient.CHUNK_SIZE / 4)));
runtimeClient.sendMessages(programRunId, topicId, messages.iterator());
// Assert that future is not blocking anymore
Assert.assertTrue(countDownLatch.await(5, TimeUnit.SECONDS));
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class TetheringRuntimeJobManagerTest method setUp.
@BeforeClass
public static void setUp() throws IOException, TopicAlreadyExistsException {
CConfiguration cConf = CConfiguration.create();
cConf.set(Constants.Tethering.TOPIC_PREFIX, "prefix-");
cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
Injector injector = Guice.createInjector(new ConfigModule(cConf), new InMemoryDiscoveryModule(), new LocalLocationModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new AbstractModule() {
@Override
protected void configure() {
bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class).in(Scopes.SINGLETON);
}
});
messagingService = injector.getInstance(MessagingService.class);
if (messagingService instanceof Service) {
((Service) messagingService).startAndWait();
}
TetheringConf conf = TetheringConf.fromProperties(PROPERTIES);
topicId = new TopicId(NamespaceId.SYSTEM.getNamespace(), cConf.get(Constants.Tethering.TOPIC_PREFIX) + TETHERED_INSTANCE_NAME);
messagingService.createTopic(new TopicMetadata(topicId, Collections.emptyMap()));
messageFetcher = new MultiThreadMessagingContext(messagingService).getMessageFetcher();
runtimeJobManager = new TetheringRuntimeJobManager(conf, cConf, messagingService);
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class CoreSchedulerServiceTest method testProgramEvents.
@Test
@Category(XSlowTests.class)
public void testProgramEvents() throws Exception {
// Deploy the app
deploy(AppWithMultipleSchedules.class, 200);
CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
TopicId programEventTopic = NamespaceId.SYSTEM.topic(cConf.get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC));
ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(cConf, messagingService);
// These notifications should not trigger the program
ProgramRunId anotherWorkflowRun = ANOTHER_WORKFLOW.run(RunIds.generate());
ArtifactId artifactId = ANOTHER_WORKFLOW.getNamespaceId().artifact("test", "1.0").toApiArtifactId();
ApplicationSpecification appSpec = new DefaultApplicationSpecification(AppWithMultipleSchedules.NAME, ApplicationId.DEFAULT_VERSION, ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
ProgramDescriptor programDescriptor = new ProgramDescriptor(anotherWorkflowRun.getParent(), appSpec);
BasicArguments systemArgs = new BasicArguments(ImmutableMap.of(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString()));
ProgramOptions programOptions = new SimpleProgramOptions(anotherWorkflowRun.getParent(), systemArgs, new BasicArguments(), false);
programStateWriter.start(anotherWorkflowRun, programOptions, null, programDescriptor);
programStateWriter.running(anotherWorkflowRun, null);
long lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
programStateWriter.error(anotherWorkflowRun, null);
waitUntilProcessed(programEventTopic, lastProcessed);
ProgramRunId someWorkflowRun = SOME_WORKFLOW.run(RunIds.generate());
programDescriptor = new ProgramDescriptor(someWorkflowRun.getParent(), appSpec);
programStateWriter.start(someWorkflowRun, new SimpleProgramOptions(someWorkflowRun.getParent(), systemArgs, new BasicArguments()), null, programDescriptor);
programStateWriter.running(someWorkflowRun, null);
lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
programStateWriter.killed(someWorkflowRun);
waitUntilProcessed(programEventTopic, lastProcessed);
Assert.assertEquals(0, getRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.ALL));
// Enable the schedule
scheduler.enableSchedule(APP_MULT_ID.schedule(AppWithMultipleSchedules.WORKFLOW_COMPLETED_SCHEDULE));
// Start a program with user arguments
startProgram(ANOTHER_WORKFLOW, ImmutableMap.of(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_KEY, AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE), 200);
// Wait for a completed run record
waitForCompleteRuns(1, TRIGGERED_WORKFLOW);
assertProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED, 1);
RunRecord run = getProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED).get(0);
Map<String, List<WorkflowTokenDetail.NodeValueDetail>> tokenData = getWorkflowToken(TRIGGERED_WORKFLOW, run.getPid(), null, null).getTokenData();
// There should be 2 entries in tokenData
Assert.assertEquals(2, tokenData.size());
// The value of TRIGGERED_RUNTIME_ARG_KEY should be ANOTHER_RUNTIME_ARG_VALUE from the triggering workflow
Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_RUNTIME_ARG_KEY).get(0).getValue());
// The value of TRIGGERED_TOKEN_KEY should be ANOTHER_TOKEN_VALUE from the triggering workflow
Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_TOKEN_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_TOKEN_KEY).get(0).getValue());
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class RuntimeHandler method writeMessages.
/**
* Handles call for writing to TMS from the program runtime for a given program run. The POST body is an
* avro array of bytes.
*/
@Path("/topics/{topic}")
@POST
public BodyConsumer writeMessages(HttpRequest request, HttpResponder responder, @PathParam("namespace") String namespace, @PathParam("app") String app, @PathParam("version") String version, @PathParam("program-type") String programType, @PathParam("program") String program, @PathParam("run") String run, @PathParam("topic") String topic) throws Exception {
if (!"avro/binary".equals(request.headers().get(HttpHeaderNames.CONTENT_TYPE))) {
throw new BadRequestException("Only avro/binary content type is supported.");
}
ApplicationId appId = new NamespaceId(namespace).app(app, version);
ProgramRunId programRunId = new ProgramRunId(appId, ProgramType.valueOfCategoryName(programType, BadRequestException::new), program, run);
ProgramRunInfo programRunInfo = requestValidator.getProgramRunStatus(programRunId, request);
if (!allowedTopics.contains(topic)) {
throw new UnauthorizedException("Access denied for topic " + topic);
}
TopicId topicId = NamespaceId.SYSTEM.topic(topic);
if (topic.startsWith(logsTopicPrefix)) {
return new MessageBodyConsumer(topicId, logProcessor::process, programRunInfo);
}
return new MessageBodyConsumer(topicId, payloads -> {
try {
messagingContext.getDirectMessagePublisher().publish(topicId.getNamespace(), topicId.getTopic(), payloads);
} catch (TopicNotFoundException e) {
throw new BadRequestException(e);
}
}, programRunInfo);
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.
the class MessagingUtilsTest method testSystemTopics.
@Test
public void testSystemTopics() {
CConfiguration cConf = CConfiguration.create();
cConf.set(Constants.MessagingSystem.SYSTEM_TOPICS, " topic-1, topic_2 ,prefix:10,invalid.name");
Set<TopicId> topics = MessagingServiceUtils.getSystemTopics(cConf, true);
Set<TopicId> expected = new LinkedHashSet<>();
expected.add(NamespaceId.SYSTEM.topic("topic-1"));
expected.add(NamespaceId.SYSTEM.topic("topic_2"));
for (int i = 0; i < 10; i++) {
expected.add(NamespaceId.SYSTEM.topic("prefix" + i));
}
Assert.assertEquals(expected, topics);
}
Aggregations