Search in sources :

Example 6 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class RuntimeClientServerTest method testFutureIsNotBlockingWhenValueIsSet.

@Test
public void testFutureIsNotBlockingWhenValueIsSet() throws Exception {
    CountDownLatch countDownLatch = new CountDownLatch(1);
    runtimeClient.onProgramStopRequested(() -> countDownLatch.countDown());
    // Now call sendMessages which will set the future
    ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("workflow").run(RunIds.generate());
    TopicId topicId = NamespaceId.SYSTEM.topic("topic");
    List<Message> messages = new ArrayList<>();
    messages.add(createMessage(Math.max(1, RuntimeClient.CHUNK_SIZE / 4)));
    runtimeClient.sendMessages(programRunId, topicId, messages.iterator());
    // Assert that future is not blocking anymore
    Assert.assertTrue(countDownLatch.await(5, TimeUnit.SECONDS));
}
Also used : Message(io.cdap.cdap.api.messaging.Message) ArrayList(java.util.ArrayList) TopicId(io.cdap.cdap.proto.id.TopicId) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.junit.Test)

Example 7 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class TetheringRuntimeJobManagerTest method setUp.

@BeforeClass
public static void setUp() throws IOException, TopicAlreadyExistsException {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.Tethering.TOPIC_PREFIX, "prefix-");
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new InMemoryDiscoveryModule(), new LocalLocationModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class).in(Scopes.SINGLETON);
        }
    });
    messagingService = injector.getInstance(MessagingService.class);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    TetheringConf conf = TetheringConf.fromProperties(PROPERTIES);
    topicId = new TopicId(NamespaceId.SYSTEM.getNamespace(), cConf.get(Constants.Tethering.TOPIC_PREFIX) + TETHERED_INSTANCE_NAME);
    messagingService.createTopic(new TopicMetadata(topicId, Collections.emptyMap()));
    messageFetcher = new MultiThreadMessagingContext(messagingService).getMessageFetcher();
    runtimeJobManager = new TetheringRuntimeJobManager(conf, cConf, messagingService);
}
Also used : InMemoryDiscoveryModule(io.cdap.cdap.common.guice.InMemoryDiscoveryModule) NoOpMetricsCollectionService(io.cdap.cdap.common.metrics.NoOpMetricsCollectionService) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) ConfigModule(io.cdap.cdap.common.guice.ConfigModule) NoOpMetricsCollectionService(io.cdap.cdap.common.metrics.NoOpMetricsCollectionService) MessagingService(io.cdap.cdap.messaging.MessagingService) Service(com.google.common.util.concurrent.Service) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) MessagingServerRuntimeModule(io.cdap.cdap.messaging.guice.MessagingServerRuntimeModule) MultiThreadMessagingContext(io.cdap.cdap.messaging.context.MultiThreadMessagingContext) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) AbstractModule(com.google.inject.AbstractModule) MessagingService(io.cdap.cdap.messaging.MessagingService) TopicMetadata(io.cdap.cdap.messaging.TopicMetadata) LocalLocationModule(io.cdap.cdap.common.guice.LocalLocationModule) TetheringConf(io.cdap.cdap.internal.tethering.runtime.spi.provisioner.TetheringConf) Injector(com.google.inject.Injector) TopicId(io.cdap.cdap.proto.id.TopicId) BeforeClass(org.junit.BeforeClass)

Example 8 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class CoreSchedulerServiceTest method testProgramEvents.

@Test
@Category(XSlowTests.class)
public void testProgramEvents() throws Exception {
    // Deploy the app
    deploy(AppWithMultipleSchedules.class, 200);
    CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
    TopicId programEventTopic = NamespaceId.SYSTEM.topic(cConf.get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC));
    ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(cConf, messagingService);
    // These notifications should not trigger the program
    ProgramRunId anotherWorkflowRun = ANOTHER_WORKFLOW.run(RunIds.generate());
    ArtifactId artifactId = ANOTHER_WORKFLOW.getNamespaceId().artifact("test", "1.0").toApiArtifactId();
    ApplicationSpecification appSpec = new DefaultApplicationSpecification(AppWithMultipleSchedules.NAME, ApplicationId.DEFAULT_VERSION, ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(anotherWorkflowRun.getParent(), appSpec);
    BasicArguments systemArgs = new BasicArguments(ImmutableMap.of(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString()));
    ProgramOptions programOptions = new SimpleProgramOptions(anotherWorkflowRun.getParent(), systemArgs, new BasicArguments(), false);
    programStateWriter.start(anotherWorkflowRun, programOptions, null, programDescriptor);
    programStateWriter.running(anotherWorkflowRun, null);
    long lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    programStateWriter.error(anotherWorkflowRun, null);
    waitUntilProcessed(programEventTopic, lastProcessed);
    ProgramRunId someWorkflowRun = SOME_WORKFLOW.run(RunIds.generate());
    programDescriptor = new ProgramDescriptor(someWorkflowRun.getParent(), appSpec);
    programStateWriter.start(someWorkflowRun, new SimpleProgramOptions(someWorkflowRun.getParent(), systemArgs, new BasicArguments()), null, programDescriptor);
    programStateWriter.running(someWorkflowRun, null);
    lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    programStateWriter.killed(someWorkflowRun);
    waitUntilProcessed(programEventTopic, lastProcessed);
    Assert.assertEquals(0, getRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.ALL));
    // Enable the schedule
    scheduler.enableSchedule(APP_MULT_ID.schedule(AppWithMultipleSchedules.WORKFLOW_COMPLETED_SCHEDULE));
    // Start a program with user arguments
    startProgram(ANOTHER_WORKFLOW, ImmutableMap.of(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_KEY, AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE), 200);
    // Wait for a completed run record
    waitForCompleteRuns(1, TRIGGERED_WORKFLOW);
    assertProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED, 1);
    RunRecord run = getProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED).get(0);
    Map<String, List<WorkflowTokenDetail.NodeValueDetail>> tokenData = getWorkflowToken(TRIGGERED_WORKFLOW, run.getPid(), null, null).getTokenData();
    // There should be 2 entries in tokenData
    Assert.assertEquals(2, tokenData.size());
    // The value of TRIGGERED_RUNTIME_ARG_KEY should be ANOTHER_RUNTIME_ARG_VALUE from the triggering workflow
    Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_RUNTIME_ARG_KEY).get(0).getValue());
    // The value of TRIGGERED_TOKEN_KEY should be ANOTHER_TOKEN_VALUE from the triggering workflow
    Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_TOKEN_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_TOKEN_KEY).get(0).getValue());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) DefaultApplicationSpecification(io.cdap.cdap.internal.app.DefaultApplicationSpecification) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) MessagingProgramStateWriter(io.cdap.cdap.internal.app.program.MessagingProgramStateWriter) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) ProgramOptions(io.cdap.cdap.app.runtime.ProgramOptions) RunRecord(io.cdap.cdap.proto.RunRecord) ProgramStateWriter(io.cdap.cdap.app.runtime.ProgramStateWriter) MessagingProgramStateWriter(io.cdap.cdap.internal.app.program.MessagingProgramStateWriter) TopicId(io.cdap.cdap.proto.id.TopicId) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) DefaultApplicationSpecification(io.cdap.cdap.internal.app.DefaultApplicationSpecification) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) WorkflowTokenDetail(io.cdap.cdap.proto.WorkflowTokenDetail) Category(org.junit.experimental.categories.Category) Test(org.junit.Test)

Example 9 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class RuntimeHandler method writeMessages.

/**
 * Handles call for writing to TMS from the program runtime for a given program run. The POST body is an
 * avro array of bytes.
 */
@Path("/topics/{topic}")
@POST
public BodyConsumer writeMessages(HttpRequest request, HttpResponder responder, @PathParam("namespace") String namespace, @PathParam("app") String app, @PathParam("version") String version, @PathParam("program-type") String programType, @PathParam("program") String program, @PathParam("run") String run, @PathParam("topic") String topic) throws Exception {
    if (!"avro/binary".equals(request.headers().get(HttpHeaderNames.CONTENT_TYPE))) {
        throw new BadRequestException("Only avro/binary content type is supported.");
    }
    ApplicationId appId = new NamespaceId(namespace).app(app, version);
    ProgramRunId programRunId = new ProgramRunId(appId, ProgramType.valueOfCategoryName(programType, BadRequestException::new), program, run);
    ProgramRunInfo programRunInfo = requestValidator.getProgramRunStatus(programRunId, request);
    if (!allowedTopics.contains(topic)) {
        throw new UnauthorizedException("Access denied for topic " + topic);
    }
    TopicId topicId = NamespaceId.SYSTEM.topic(topic);
    if (topic.startsWith(logsTopicPrefix)) {
        return new MessageBodyConsumer(topicId, logProcessor::process, programRunInfo);
    }
    return new MessageBodyConsumer(topicId, payloads -> {
        try {
            messagingContext.getDirectMessagePublisher().publish(topicId.getNamespace(), topicId.getTopic(), payloads);
        } catch (TopicNotFoundException e) {
            throw new BadRequestException(e);
        }
    }, programRunInfo);
}
Also used : TopicNotFoundException(io.cdap.cdap.api.messaging.TopicNotFoundException) UnauthorizedException(io.cdap.cdap.security.spi.authorization.UnauthorizedException) BadRequestException(io.cdap.cdap.common.BadRequestException) TopicId(io.cdap.cdap.proto.id.TopicId) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Path(javax.ws.rs.Path) POST(javax.ws.rs.POST)

Example 10 with TopicId

use of io.cdap.cdap.proto.id.TopicId in project cdap by caskdata.

the class MessagingUtilsTest method testSystemTopics.

@Test
public void testSystemTopics() {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.MessagingSystem.SYSTEM_TOPICS, "  topic-1, topic_2 ,prefix:10,invalid.name");
    Set<TopicId> topics = MessagingServiceUtils.getSystemTopics(cConf, true);
    Set<TopicId> expected = new LinkedHashSet<>();
    expected.add(NamespaceId.SYSTEM.topic("topic-1"));
    expected.add(NamespaceId.SYSTEM.topic("topic_2"));
    for (int i = 0; i < 10; i++) {
        expected.add(NamespaceId.SYSTEM.topic("prefix" + i));
    }
    Assert.assertEquals(expected, topics);
}
Also used : LinkedHashSet(java.util.LinkedHashSet) TopicId(io.cdap.cdap.proto.id.TopicId) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) Test(org.junit.Test)

Aggregations

TopicId (io.cdap.cdap.proto.id.TopicId)152 Test (org.junit.Test)84 TopicMetadata (io.cdap.cdap.messaging.TopicMetadata)80 ArrayList (java.util.ArrayList)60 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)44 RawMessage (io.cdap.cdap.messaging.data.RawMessage)32 TopicNotFoundException (io.cdap.cdap.api.messaging.TopicNotFoundException)28 IOException (java.io.IOException)26 MessageId (io.cdap.cdap.messaging.data.MessageId)24 Path (javax.ws.rs.Path)24 HashMap (java.util.HashMap)16 TopicAlreadyExistsException (io.cdap.cdap.api.messaging.TopicAlreadyExistsException)14 ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)14 Message (io.cdap.cdap.api.messaging.Message)12 BadRequestException (io.cdap.cdap.common.BadRequestException)12 Injector (com.google.inject.Injector)10 MessagingService (io.cdap.cdap.messaging.MessagingService)10 StoreRequest (io.cdap.cdap.messaging.StoreRequest)10 POST (javax.ws.rs.POST)10 NoOpMetricsCollectionService (io.cdap.cdap.common.metrics.NoOpMetricsCollectionService)8