Examples with Notification io.cdap.cdap.proto.Notification used on opensource projects

Example 6 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class JobQueueTableTest method testJobTimeout.

@Test
public void testJobTimeout() {
    TransactionRunners.run(transactionRunner, context -> {
        JobQueueTable jobQueue = JobQueueTable.getJobQueue(context, getCConf());
        // should be 0 jobs in the JobQueue to begin with
        Assert.assertEquals(0, getAllJobs(jobQueue, false).size());
        // Construct a partition notification with DATASET_ID
        Notification notification = Notification.forPartitions(DATASET_ID, ImmutableList.of());
        Assert.assertNull(jobQueue.getJob(SCHED1_JOB.getJobKey()));
        ProgramSchedule scheduleWithTimeout = new ProgramSchedule("SCHED1", "one partition schedule", WORKFLOW_ID, ImmutableMap.of("prop3", "abc"), new PartitionTrigger(DATASET_ID, 1), ImmutableList.of());
        Job jobWithTimeout = new SimpleJob(scheduleWithTimeout, 0, System.currentTimeMillis() - (Schedulers.JOB_QUEUE_TIMEOUT_MILLIS + 1), Lists.newArrayList(), Job.State.PENDING_TRIGGER, 0L);
        jobQueue.put(jobWithTimeout);
        Assert.assertEquals(jobWithTimeout, jobQueue.getJob(jobWithTimeout.getJobKey()));
        // before adding the notification, there should just be the job we added
        Assert.assertEquals(1, toSet(jobQueue.getJobsForSchedule(scheduleWithTimeout.getScheduleId()), true).size());
        // adding a notification will ignore the existing job (because it is timed out). It will create a new job
        // and add the notification to that new job
        jobQueue.addNotification(new ProgramScheduleRecord(SCHED1, new ProgramScheduleMeta(ProgramScheduleStatus.SCHEDULED, 0L)), notification);
        List<Job> jobs = new ArrayList<>(toSet(jobQueue.getJobsForSchedule(scheduleWithTimeout.getScheduleId()), true));
        // sort by creation time (oldest will be first in the list)
        Collections.sort(jobs, (o1, o2) -> Long.valueOf(o1.getCreationTime()).compareTo(o2.getCreationTime()));
        Assert.assertEquals(2, jobs.size());
        Job firstJob = jobs.get(0);
        // first job should have the same creation timestamp as the initially added job
        Assert.assertEquals(jobWithTimeout.getCreationTime(), firstJob.getCreationTime());
        // the notification we added shouldn't be in the first job
        Assert.assertEquals(0, firstJob.getNotifications().size());
        // first job should be marked to be deleted because it timed out
        Assert.assertTrue(firstJob.isToBeDeleted());
        // first job should have the same generation id as the timed out job
        Assert.assertEquals(jobWithTimeout.getGenerationId(), firstJob.getGenerationId());
        Job secondJob = jobs.get(1);
        // first job should not have the same creation timestamp as the initially added job
        Assert.assertNotEquals(jobWithTimeout.getCreationTime(), secondJob.getCreationTime());
        // the notification we added shouldn't be in the first job
        Assert.assertEquals(1, secondJob.getNotifications().size());
        Assert.assertEquals(notification, secondJob.getNotifications().get(0));
        // first job should not be marked to be deleted, since it was just created by our call to
        // JobQueue#addNotification
        Assert.assertFalse(secondJob.isToBeDeleted());
        // second job should have the next generation id
        Assert.assertEquals(jobWithTimeout.getGenerationId() + 1, secondJob.getGenerationId());
    });
}

Example 7 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class JobQueueTableTest method testAddNotifications.

@Test
public void testAddNotifications() {
    TransactionRunners.run(transactionRunner, context -> {
        JobQueueTable jobQueue = JobQueueTable.getJobQueue(context, getCConf());
        // should be 0 jobs in the JobQueue to begin with
        Assert.assertEquals(0, getAllJobs(jobQueue, false).size());
        // Construct a partition notification with DATASET_ID
        Notification notification = Notification.forPartitions(DATASET_ID, ImmutableList.of());
        Assert.assertNull(jobQueue.getJob(SCHED1_JOB.getJobKey()));
        jobQueue.put(SCHED1_JOB);
        Assert.assertEquals(SCHED1_JOB, jobQueue.getJob(SCHED1_JOB.getJobKey()));
        // Since notification and SCHED1 have the same dataset id DATASET_ID, notification will be added to
        // SCHED1_JOB, which is a job in SCHED1
        jobQueue.addNotification(new ProgramScheduleRecord(SCHED1, new ProgramScheduleMeta(ProgramScheduleStatus.SCHEDULED, 0L)), notification);
        Assert.assertEquals(ImmutableList.of(notification), jobQueue.getJob(SCHED1_JOB.getJobKey()).getNotifications());
    });
}

Example 8 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class ProvisionerNotifier method publish.

private void publish(Map<String, String> properties) {
    final StoreRequest storeRequest = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(new Notification(Notification.Type.PROGRAM_STATUS, properties))).build();
    Retries.supplyWithRetries(() -> {
        try {
            messagingService.publish(storeRequest);
        } catch (TopicNotFoundException e) {
            throw new RetryableException(e);
        } catch (IOException | AccessException e) {
            throw Throwables.propagate(e);
        }
        return null;
    }, retryStrategy);
}

Example 9 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class ProgramNotificationSubscriberService method processMessages.

@Override
protected void processMessages(StructuredTableContext structuredTableContext, Iterator<ImmutablePair<String, Notification>> messages) throws Exception {
    ProgramHeartbeatTable heartbeatDataset = new ProgramHeartbeatTable(structuredTableContext);
    List<Runnable> tasks = new LinkedList<>();
    while (messages.hasNext()) {
        ImmutablePair<String, Notification> messagePair = messages.next();
        List<Runnable> runnables = processNotification(heartbeatDataset, messagePair.getFirst().getBytes(StandardCharsets.UTF_8), messagePair.getSecond(), structuredTableContext);
        tasks.addAll(runnables);
    }
    // Only add post processing tasks if all messages are processed. If there is exception in the processNotifiation,
    // messages will be replayed.
    this.tasks.addAll(tasks);
}

Example 10 with Notification

use of io.cdap.cdap.proto.Notification in project cdap by caskdata.

the class ProgramNotificationSubscriberService method publishRecordedStatus.

private void publishRecordedStatus(Notification notification, ProgramRunId programRunId, ProgramRunStatus status) throws Exception {
    Map<String, String> notificationProperties = new HashMap<>(notification.getProperties());
    notificationProperties.put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(programRunId));
    notificationProperties.put(ProgramOptionConstants.PROGRAM_STATUS, status.name());
    notificationProperties.put(CDAP_VERSION, ProjectInfo.getVersion().toString());
    Notification programStatusNotification = new Notification(Notification.Type.PROGRAM_STATUS, notificationProperties);
    getMessagingContext().getMessagePublisher().publish(NamespaceId.SYSTEM.getNamespace(), recordedProgramStatusPublishTopic, GSON.toJson(programStatusNotification));
}