Examples with Notification io.cdap.cdap.proto.Notification used on opensource projects

Example 1 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class MessagingProgramStatePublisher method publish.

public void publish(Notification.Type notificationType, Map<String, String> properties) {
    // ProgramRunId is always required in a notification
    Notification programStatusNotification = new Notification(notificationType, properties);
    int failureCount = 0;
    long startTime = -1L;
    boolean done = false;
    // This should be refactored into a common class for publishing to TMS with a retry strategy
    while (!done) {
        try {
            messagingService.publish(StoreRequestBuilder.of(topicId).addPayload(GSON.toJson(programStatusNotification)).build());
            LOG.trace("Published program status notification: {}", programStatusNotification);
            done = true;
        } catch (IOException | AccessException e) {
            throw Throwables.propagate(e);
        } catch (TopicNotFoundException | ServiceUnavailableException e) {
            // These exceptions are retry-able due to TMS not completely started
            if (startTime < 0) {
                startTime = System.currentTimeMillis();
            }
            long retryMillis = retryStrategy.nextRetry(++failureCount, startTime);
            if (retryMillis < 0) {
                LOG.error("Failed to publish messages to TMS and exceeded retry limit.", e);
                throw Throwables.propagate(e);
            }
            LOG.debug("Failed to publish messages to TMS due to {}. Will be retried in {} ms.", e.getMessage(), retryMillis);
            try {
                TimeUnit.MILLISECONDS.sleep(retryMillis);
            } catch (InterruptedException e1) {
                // Something explicitly stopping this thread. Simply just break and reset the interrupt flag.
                LOG.warn("Publishing message to TMS interrupted.");
                Thread.currentThread().interrupt();
                done = true;
            }
        }
    }
}

Example 2 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class RuntimeClientServiceTest method testBasicRelay.

@Test
public void testBasicRelay() throws Exception {
    // Send some messages to multiple topics in the client side TMS, they should get replicated to the server side TMS.
    MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
    MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
    ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        // the RuntimeClientService will decode it to watch for program termination
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Write a non-terminal state to test basic relaying
            programStateWriter.running(PROGRAM_RUN_ID, null);
        } else {
            messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), entry.getKey(), entry.getKey());
        }
    }
    MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Extract the program run status from the Notification
            Tasks.waitFor(Collections.singletonList(ProgramRunStatus.RUNNING), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        } else {
            Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        }
    }
    // Writes a program terminate message to unblock stopping of the client service
    programStateWriter.completed(PROGRAM_RUN_ID);
}

Example 3 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class RuntimeClientServiceTest method testProgramTerminate.

/**
 * Test for {@link RuntimeClientService} that will terminate itself when seeing program completed message.
 */
@Test
public void testProgramTerminate() throws Exception {
    MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
    MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
    ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
    // Send a terminate program state first, wait for the service sees the state change,
    // then publish messages to other topics.
    programStateWriter.completed(PROGRAM_RUN_ID);
    Tasks.waitFor(true, () -> runtimeClientService.getProgramFinishTime() >= 0, 2, TimeUnit.SECONDS);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        // the RuntimeClientService will decode it to watch for program termination
        if (!entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            List<String> payloads = Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey());
            messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), StandardCharsets.UTF_8, payloads.iterator());
        }
    }
    // The client service should get stopped by itself.
    Tasks.waitFor(Service.State.TERMINATED, () -> runtimeClientService.state(), clientCConf.getLong(Constants.RuntimeMonitor.GRACEFUL_SHUTDOWN_MS) + 2000, TimeUnit.MILLISECONDS);
    // All messages should be sent after the runtime client service stopped
    MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Extract the program run status from the Notification
            Tasks.waitFor(Collections.singletonList(ProgramRunStatus.COMPLETED), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        } else {
            Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        }
    }
}

Example 4 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class MapReduceRunnerTestBase method getDataNotifications.

/**
 * Returns a list of {@link Notification} object fetched from the data event topic in TMS that was published
 * starting from the given time.
 */
protected List<Notification> getDataNotifications(long startTime) throws Exception {
    // Get data notifications from TMS
    List<Notification> notifications = new ArrayList<>();
    MessagingContext messagingContext = new MultiThreadMessagingContext(injector.getInstance(MessagingService.class));
    try (CloseableIterator<Message> messages = messagingContext.getMessageFetcher().fetch(NamespaceId.SYSTEM.getNamespace(), injector.getInstance(CConfiguration.class).get(Constants.Dataset.DATA_EVENT_TOPIC), 10, startTime)) {
        while (messages.hasNext()) {
            notifications.add(GSON.fromJson(new String(messages.next().getPayload(), StandardCharsets.UTF_8), Notification.class));
        }
    }
    return notifications;
}

Example 5 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class JobQueueTableTest method addNotificationToSchedule.

private void addNotificationToSchedule(ProgramSchedule programSchedule) {
    TransactionRunners.run(transactionRunner, context -> {
        JobQueueTable jobQueue = JobQueueTable.getJobQueue(context, getCConf());
        // Construct a partition notification with DATASET_ID
        Notification notification = Notification.forPartitions(DATASET_ID, ImmutableList.of());
        jobQueue.addNotification(new ProgramScheduleRecord(programSchedule, new ProgramScheduleMeta(ProgramScheduleStatus.SCHEDULED, 0L)), notification);
    });
}