Examples with Notification io.cdap.cdap.proto.Notification used on opensource projects

Example 36 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class RuntimeServiceMainTest method createProgramStateWriter.

/**
 * Creates a {@link ProgramStateWriter} that writes to {@link RuntimeClient} directly.
 *
 * @param injector the injector for creating the {@link RuntimeClient}
 * @param programRunId the {@link ProgramRunId} for the program state change
 * @return a {@link ProgramStateWriter}
 */
private ProgramStateWriter createProgramStateWriter(Injector injector, ProgramRunId programRunId) {
    RuntimeClient runtimeClient = injector.getInstance(RuntimeClient.class);
    // We write to the record event directly to skip the app-fabric to process it
    // This is because we don't follow the normal event flow here for testing
    TopicId topicId = NamespaceId.SYSTEM.topic(injector.getInstance(CConfiguration.class).get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC));
    RetryStrategy retryStrategy = RetryStrategies.timeLimit(5, TimeUnit.SECONDS, RetryStrategies.fixDelay(200, TimeUnit.MILLISECONDS));
    return new MessagingProgramStateWriter((notificationType, properties) -> {
        Notification notification = new Notification(notificationType, properties);
        try {
            Retries.callWithRetries((Retries.Callable<Void, Exception>) () -> {
                runtimeClient.sendMessages(programRunId, topicId, Collections.singleton(createMessage(notification)).iterator());
                return null;
            }, retryStrategy, t -> t instanceof IOException || t instanceof RetryableException);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    });
}

Example 37 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class CoreSchedulerServiceTest method testScheduleUpdate.

private void testScheduleUpdate(String howToUpdate) throws Exception {
    int runs = getRuns(WORKFLOW_2, ProgramRunStatus.ALL);
    final ScheduleId scheduleId2 = APP_ID.schedule(AppWithFrequentScheduledWorkflows.DATASET_PARTITION_SCHEDULE_2);
    // send one notification to it
    long minPublishTime = System.currentTimeMillis();
    publishNotification(dataEventTopic, NamespaceId.DEFAULT, AppWithFrequentScheduledWorkflows.DATASET_NAME2);
    waitUntilProcessed(dataEventTopic, minPublishTime);
    // A pending job will be created, but it won't run
    Assert.assertTrue("Expected a PENDING_TRIGGER job for " + scheduleId2, Iterables.any(getAllJobs(), job -> {
        if (!(job.getSchedule().getTrigger() instanceof ProtoTrigger.PartitionTrigger)) {
            return false;
        }
        return scheduleId2.equals(job.getJobKey().getScheduleId()) && job.getState() == Job.State.PENDING_TRIGGER;
    }));
    Assert.assertEquals(runs, getRuns(WORKFLOW_2, ProgramRunStatus.ALL));
    if ("disable".equals(howToUpdate)) {
        // disabling and enabling the schedule should remove the job
        disableSchedule(AppWithFrequentScheduledWorkflows.DATASET_PARTITION_SCHEDULE_2);
        enableSchedule(AppWithFrequentScheduledWorkflows.DATASET_PARTITION_SCHEDULE_2);
    } else {
        ProgramSchedule schedule = scheduler.getSchedule(scheduleId2);
        Map<String, String> updatedProperties = ImmutableMap.<String, String>builder().putAll(schedule.getProperties()).put(howToUpdate, howToUpdate).build();
        ProgramSchedule updatedSchedule = new ProgramSchedule(schedule.getName(), schedule.getDescription(), schedule.getProgramId(), updatedProperties, schedule.getTrigger(), schedule.getConstraints());
        if ("update".equals(howToUpdate)) {
            scheduler.updateSchedule(updatedSchedule);
            Assert.assertEquals(ProgramScheduleStatus.SCHEDULED, scheduler.getScheduleStatus(scheduleId2));
        } else if ("delete".equals(howToUpdate)) {
            scheduler.deleteSchedule(scheduleId2);
            scheduler.addSchedule(updatedSchedule);
            enableSchedule(scheduleId2.getSchedule());
        } else {
            Assert.fail("invalid howToUpdate: " + howToUpdate);
        }
    }
    // single notification should not trigger workflow 2 yet (if it does, then the job was not removed)
    minPublishTime = System.currentTimeMillis();
    publishNotification(dataEventTopic, NamespaceId.DEFAULT, AppWithFrequentScheduledWorkflows.DATASET_NAME2);
    waitUntilProcessed(dataEventTopic, minPublishTime);
    // Again, a pending job will be created, but it won't run since updating the schedule would remove pending trigger
    Assert.assertTrue("Expected a PENDING_TRIGGER job for " + scheduleId2, Iterables.any(getAllJobs(), job -> {
        if (!(job.getSchedule().getTrigger() instanceof ProtoTrigger.PartitionTrigger)) {
            return false;
        }
        return scheduleId2.equals(job.getJobKey().getScheduleId()) && job.getState() == Job.State.PENDING_TRIGGER;
    }));
    Assert.assertEquals(runs, getRuns(WORKFLOW_2, ProgramRunStatus.ALL));
    // publish one more notification, this should kick off the workflow
    publishNotification(dataEventTopic, NamespaceId.DEFAULT, AppWithFrequentScheduledWorkflows.DATASET_NAME2);
    waitForCompleteRuns(runs + 1, WORKFLOW_2);
}

Example 38 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class CoreSchedulerServiceTest method publishNotification.

private void publishNotification(TopicId topicId, NamespaceId namespaceId, String dataset) throws Exception {
    DatasetId datasetId = namespaceId.dataset(dataset);
    PartitionKey partitionKey = PartitionKey.builder().addIntField("part1", 1).build();
    Notification notification = Notification.forPartitions(datasetId, ImmutableList.of(partitionKey));
    messagingService.publish(StoreRequestBuilder.of(topicId).addPayload(GSON.toJson(notification)).build());
}

Example 39 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class ProgramStatusEventPublisherTest method provideMockedMessages.

private Iterator<ImmutablePair<String, Notification>> provideMockedMessages() {
    ClassLoader classLoader = ClassLoader.getSystemClassLoader();
    InputStream notificationIS = classLoader.getResourceAsStream(MOCKED_NOTIFICATION_FILENAME);
    Assert.assertNotNull(notificationIS);
    String notificationJson = new BufferedReader(new InputStreamReader(notificationIS)).lines().collect(Collectors.joining(System.lineSeparator()));
    Notification notification = GSON.fromJson(notificationJson, Notification.class);
    ImmutablePair<String, Notification> message = new ImmutablePair<>("test", notification);
    List<ImmutablePair<String, Notification>> messageList = new ArrayList<>();
    messageList.add(message);
    return messageList.iterator();
}

Example 40 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class ProgramStatusEventPublisher method processMessages.

@Override
protected void processMessages(StructuredTableContext structuredTableContext, Iterator<ImmutablePair<String, Notification>> messages) {
    List<ProgramStatusEvent> programStatusEvents = new ArrayList<>();
    long publishTime = System.currentTimeMillis();
    messages.forEachRemaining(message -> {
        Notification notification = message.getSecond();
        if (!notification.getNotificationType().equals(Notification.Type.PROGRAM_STATUS)) {
            return;
        }
        Map<String, String> properties = notification.getProperties();
        // get program run ID
        String programStatus = properties.get(ProgramOptionConstants.PROGRAM_STATUS);
        if (programStatus == null) {
            return;
        }
        ProgramRunStatus programRunStatus = ProgramRunStatus.valueOf(programStatus);
        String programRun = properties.get(ProgramOptionConstants.PROGRAM_RUN_ID);
        ProgramRunId programRunId = GSON.fromJson(programRun, ProgramRunId.class);
        // Should event publish happen for this status
        if (!shouldPublish(programRunId)) {
            return;
        }
        ProgramStatusEventDetails.Builder builder = ProgramStatusEventDetails.getBuilder(programRunId.getRun(), programRunId.getApplication(), programRunId.getProgram(), programRunId.getNamespace(), programStatus, RunIds.getTime(programRunId.getRun(), TimeUnit.MILLISECONDS));
        String userArgsString = properties.get(ProgramOptionConstants.USER_OVERRIDES);
        String sysArgsString = properties.get(ProgramOptionConstants.SYSTEM_OVERRIDES);
        Type argsMapType = new TypeToken<Map<String, String>>() {
        }.getType();
        builder = builder.withUserArgs(GSON.fromJson(userArgsString, argsMapType)).withSystemArgs(GSON.fromJson(sysArgsString, argsMapType));
        if (programRunStatus.isEndState()) {
            builder = populateErrorDetailsAndMetrics(builder, properties, programRunStatus, programRunId);
        }
        ProgramStatusEventDetails programStatusEventDetails = builder.build();
        ProgramStatusEvent programStatusEvent = new ProgramStatusEvent(publishTime, EVENT_VERSION, instanceName, projectName, programStatusEventDetails);
        programStatusEvents.add(programStatusEvent);
    });
    this.eventWriters.forEach(eventWriter -> eventWriter.write(programStatusEvents));
}