Examples with MessagePublisher io.cdap.cdap.api.messaging.MessagePublisher used on opensource projects

Example 1 with MessagePublisher

use of io.cdap.cdap.api.messaging.MessagePublisher in project cdap by caskdata.

the class RuntimeClientServiceTest method testBasicRelay.

@Test
public void testBasicRelay() throws Exception {
    // Send some messages to multiple topics in the client side TMS, they should get replicated to the server side TMS.
    MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
    MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
    ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        // the RuntimeClientService will decode it to watch for program termination
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Write a non-terminal state to test basic relaying
            programStateWriter.running(PROGRAM_RUN_ID, null);
        } else {
            messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), entry.getKey(), entry.getKey());
        }
    }
    MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Extract the program run status from the Notification
            Tasks.waitFor(Collections.singletonList(ProgramRunStatus.RUNNING), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        } else {
            Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        }
    }
    // Writes a program terminate message to unblock stopping of the client service
    programStateWriter.completed(PROGRAM_RUN_ID);
}

Example 2 with MessagePublisher

use of io.cdap.cdap.api.messaging.MessagePublisher in project cdap by caskdata.

the class RuntimeClientServiceTest method testProgramTerminate.

/**
 * Test for {@link RuntimeClientService} that will terminate itself when seeing program completed message.
 */
@Test
public void testProgramTerminate() throws Exception {
    MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
    MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
    ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
    // Send a terminate program state first, wait for the service sees the state change,
    // then publish messages to other topics.
    programStateWriter.completed(PROGRAM_RUN_ID);
    Tasks.waitFor(true, () -> runtimeClientService.getProgramFinishTime() >= 0, 2, TimeUnit.SECONDS);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        // the RuntimeClientService will decode it to watch for program termination
        if (!entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            List<String> payloads = Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey());
            messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), StandardCharsets.UTF_8, payloads.iterator());
        }
    }
    // The client service should get stopped by itself.
    Tasks.waitFor(Service.State.TERMINATED, () -> runtimeClientService.state(), clientCConf.getLong(Constants.RuntimeMonitor.GRACEFUL_SHUTDOWN_MS) + 2000, TimeUnit.MILLISECONDS);
    // All messages should be sent after the runtime client service stopped
    MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Extract the program run status from the Notification
            Tasks.waitFor(Collections.singletonList(ProgramRunStatus.COMPLETED), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        } else {
            Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        }
    }
}

Example 3 with MessagePublisher

use of io.cdap.cdap.api.messaging.MessagePublisher in project cdap by caskdata.

the class AbstractContext method createRuntimeProgramContext.

/**
 * Creates a new instance of {@link RuntimeProgramContext} to be
 * provided to {@link RuntimeProgramContextAware} dataset.
 */
private RuntimeProgramContext createRuntimeProgramContext(final DatasetId datasetId) {
    return new RuntimeProgramContext() {

        @Override
        public void notifyNewPartitions(Collection<? extends PartitionKey> partitionKeys) throws IOException {
            String topic = cConf.get(Constants.Dataset.DATA_EVENT_TOPIC);
            if (Strings.isNullOrEmpty(topic)) {
                // Don't publish if there is no data event topic
                return;
            }
            TopicId dataEventTopic = NamespaceId.SYSTEM.topic(topic);
            MessagePublisher publisher = getMessagingContext().getMessagePublisher();
            byte[] payload = Bytes.toBytes(GSON.toJson(Notification.forPartitions(datasetId, partitionKeys)));
            int failure = 0;
            long startTime = System.currentTimeMillis();
            while (true) {
                try {
                    publisher.publish(dataEventTopic.getNamespace(), dataEventTopic.getTopic(), payload);
                    return;
                } catch (TopicNotFoundException e) {
                    // this shouldn't happen since the TMS creates the data event topic on startup.
                    throw new IOException("Unexpected exception due to missing topic '" + dataEventTopic + "'", e);
                } catch (AccessException e) {
                    throw new IOException("Unexpected access exception during publishing notification to '" + dataEventTopic + "'", e);
                } catch (IOException e) {
                    long sleepTime = retryStrategy.nextRetry(++failure, startTime);
                    if (sleepTime < 0) {
                        throw e;
                    }
                    try {
                        TimeUnit.MILLISECONDS.sleep(sleepTime);
                    } catch (InterruptedException ex) {
                        // If interrupted during sleep, just reset the interrupt flag and return
                        Thread.currentThread().interrupt();
                        return;
                    }
                }
            }
        }

        @Override
        public ProgramRunId getProgramRunId() {
            return programRunId;
        }

        @Nullable
        @Override
        public NamespacedEntityId getComponentId() {
            return AbstractContext.this.getComponentId();
        }
    };
}

Example 4 with MessagePublisher

use of io.cdap.cdap.api.messaging.MessagePublisher in project cdap by caskdata.

the class MessagingAppTestRun method testWithWorker.

@Test
public void testWithWorker() throws Exception {
    ApplicationManager appManager = deployWithArtifact(NAMESPACE, MessagingApp.class, artifactJar);
    final WorkerManager workerManager = appManager.getWorkerManager(MessagingApp.MessagingWorker.class.getSimpleName()).start();
    MessagingContext messagingContext = getMessagingContext();
    final MessagingAdmin messagingAdmin = getMessagingAdmin(NAMESPACE);
    // Wait for the worker to create the topic
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            try {
                messagingAdmin.getTopicProperties(MessagingApp.TOPIC);
                return true;
            } catch (TopicNotFoundException e) {
                return false;
            }
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Publish a message
    String message = "message";
    MessagePublisher messagePublisher = messagingContext.getMessagePublisher();
    messagePublisher.publish(NAMESPACE.getNamespace(), MessagingApp.TOPIC, message);
    // The worker will publish back a message with payload as concat(message, message)
    final MessageFetcher messageFetcher = messagingContext.getMessageFetcher();
    Tasks.waitFor(message + message, new Callable<String>() {

        @Override
        public String call() throws Exception {
            try (CloseableIterator<Message> iterator = messageFetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, Integer.MAX_VALUE, 0L)) {
                Message message = Iterators.getLast(iterator, null);
                return message == null ? null : message.getPayloadAsString();
            }
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Publish concat(message + message) to the app
    messagePublisher.publish(NAMESPACE.getNamespace(), MessagingApp.TOPIC, message + message);
    // timeout.
    try {
        Tasks.waitFor(message + message + message + message, new Callable<String>() {

            @Override
            public String call() throws Exception {
                try (CloseableIterator<Message> iterator = messageFetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, Integer.MAX_VALUE, 0L)) {
                    Message message = Iterators.getLast(iterator, null);
                    return message == null ? null : message.getPayloadAsString();
                }
            }
        }, 2, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        Assert.fail("Expected timeout exception");
    } catch (TimeoutException e) {
    // expected
    }
    // Now publish a message to the control topic, to unblock the transaction block.
    messagePublisher.publish(NAMESPACE.getNamespace(), MessagingApp.CONTROL_TOPIC, message);
    // Should expect a new message as concat(message, message, message, message)
    Tasks.waitFor(message + message + message + message, new Callable<String>() {

        @Override
        public String call() throws Exception {
            try (CloseableIterator<Message> iterator = messageFetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, Integer.MAX_VALUE, 0L)) {
                Message message = Iterators.getLast(iterator, null);
                return message == null ? null : message.getPayloadAsString();
            }
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Wait for the worker to finish and verify that it completes successfully.
    workerManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.SECONDS);
}