Examples with ProgramScheduleMeta io.cdap.cdap.internal.app.runtime.schedule.ProgramScheduleMeta used on opensource projects

Example 1 with ProgramScheduleMeta

use of io.cdap.cdap.internal.app.runtime.schedule.ProgramScheduleMeta in project cdap by caskdata.

the class JobQueueTableTest method addNotificationToSchedule.

private void addNotificationToSchedule(ProgramSchedule programSchedule) {
    TransactionRunners.run(transactionRunner, context -> {
        JobQueueTable jobQueue = JobQueueTable.getJobQueue(context, getCConf());
        // Construct a partition notification with DATASET_ID
        Notification notification = Notification.forPartitions(DATASET_ID, ImmutableList.of());
        jobQueue.addNotification(new ProgramScheduleRecord(programSchedule, new ProgramScheduleMeta(ProgramScheduleStatus.SCHEDULED, 0L)), notification);
    });
}

Example 2 with ProgramScheduleMeta

use of io.cdap.cdap.internal.app.runtime.schedule.ProgramScheduleMeta in project cdap by caskdata.

the class JobQueueTableTest method testJobTimeout.

@Test
public void testJobTimeout() {
    TransactionRunners.run(transactionRunner, context -> {
        JobQueueTable jobQueue = JobQueueTable.getJobQueue(context, getCConf());
        // should be 0 jobs in the JobQueue to begin with
        Assert.assertEquals(0, getAllJobs(jobQueue, false).size());
        // Construct a partition notification with DATASET_ID
        Notification notification = Notification.forPartitions(DATASET_ID, ImmutableList.of());
        Assert.assertNull(jobQueue.getJob(SCHED1_JOB.getJobKey()));
        ProgramSchedule scheduleWithTimeout = new ProgramSchedule("SCHED1", "one partition schedule", WORKFLOW_ID, ImmutableMap.of("prop3", "abc"), new PartitionTrigger(DATASET_ID, 1), ImmutableList.of());
        Job jobWithTimeout = new SimpleJob(scheduleWithTimeout, 0, System.currentTimeMillis() - (Schedulers.JOB_QUEUE_TIMEOUT_MILLIS + 1), Lists.newArrayList(), Job.State.PENDING_TRIGGER, 0L);
        jobQueue.put(jobWithTimeout);
        Assert.assertEquals(jobWithTimeout, jobQueue.getJob(jobWithTimeout.getJobKey()));
        // before adding the notification, there should just be the job we added
        Assert.assertEquals(1, toSet(jobQueue.getJobsForSchedule(scheduleWithTimeout.getScheduleId()), true).size());
        // adding a notification will ignore the existing job (because it is timed out). It will create a new job
        // and add the notification to that new job
        jobQueue.addNotification(new ProgramScheduleRecord(SCHED1, new ProgramScheduleMeta(ProgramScheduleStatus.SCHEDULED, 0L)), notification);
        List<Job> jobs = new ArrayList<>(toSet(jobQueue.getJobsForSchedule(scheduleWithTimeout.getScheduleId()), true));
        // sort by creation time (oldest will be first in the list)
        Collections.sort(jobs, (o1, o2) -> Long.valueOf(o1.getCreationTime()).compareTo(o2.getCreationTime()));
        Assert.assertEquals(2, jobs.size());
        Job firstJob = jobs.get(0);
        // first job should have the same creation timestamp as the initially added job
        Assert.assertEquals(jobWithTimeout.getCreationTime(), firstJob.getCreationTime());
        // the notification we added shouldn't be in the first job
        Assert.assertEquals(0, firstJob.getNotifications().size());
        // first job should be marked to be deleted because it timed out
        Assert.assertTrue(firstJob.isToBeDeleted());
        // first job should have the same generation id as the timed out job
        Assert.assertEquals(jobWithTimeout.getGenerationId(), firstJob.getGenerationId());
        Job secondJob = jobs.get(1);
        // first job should not have the same creation timestamp as the initially added job
        Assert.assertNotEquals(jobWithTimeout.getCreationTime(), secondJob.getCreationTime());
        // the notification we added shouldn't be in the first job
        Assert.assertEquals(1, secondJob.getNotifications().size());
        Assert.assertEquals(notification, secondJob.getNotifications().get(0));
        // first job should not be marked to be deleted, since it was just created by our call to
        // JobQueue#addNotification
        Assert.assertFalse(secondJob.isToBeDeleted());
        // second job should have the next generation id
        Assert.assertEquals(jobWithTimeout.getGenerationId() + 1, secondJob.getGenerationId());
    });
}

Example 3 with ProgramScheduleMeta

use of io.cdap.cdap.internal.app.runtime.schedule.ProgramScheduleMeta in project cdap by caskdata.

the class JobQueueTableTest method testAddNotifications.

@Test
public void testAddNotifications() {
    TransactionRunners.run(transactionRunner, context -> {
        JobQueueTable jobQueue = JobQueueTable.getJobQueue(context, getCConf());
        // should be 0 jobs in the JobQueue to begin with
        Assert.assertEquals(0, getAllJobs(jobQueue, false).size());
        // Construct a partition notification with DATASET_ID
        Notification notification = Notification.forPartitions(DATASET_ID, ImmutableList.of());
        Assert.assertNull(jobQueue.getJob(SCHED1_JOB.getJobKey()));
        jobQueue.put(SCHED1_JOB);
        Assert.assertEquals(SCHED1_JOB, jobQueue.getJob(SCHED1_JOB.getJobKey()));
        // Since notification and SCHED1 have the same dataset id DATASET_ID, notification will be added to
        // SCHED1_JOB, which is a job in SCHED1
        jobQueue.addNotification(new ProgramScheduleRecord(SCHED1, new ProgramScheduleMeta(ProgramScheduleStatus.SCHEDULED, 0L)), notification);
        Assert.assertEquals(ImmutableList.of(notification), jobQueue.getJob(SCHED1_JOB.getJobKey()).getNotifications());
    });
}

Example 4 with ProgramScheduleMeta

use of io.cdap.cdap.internal.app.runtime.schedule.ProgramScheduleMeta in project cdap by caskdata.

the class ProgramScheduleStoreDataset method findSchedules.

/**
 * Find all schedules that have a trigger with a given trigger key.
 *
 * @param triggerKey the trigger key to look up
 * @return a list of all schedules that are triggered by this key; never null
 */
public Collection<ProgramScheduleRecord> findSchedules(String triggerKey) throws IOException {
    Map<ScheduleId, ProgramScheduleRecord> schedulesFound = new HashMap<>();
    Field<String> triggerField = Fields.stringField(StoreDefinition.ProgramScheduleStore.TRIGGER_KEY, triggerKey);
    try (CloseableIterator<StructuredRow> iterator = triggerStore.scan(triggerField)) {
        while (iterator.hasNext()) {
            StructuredRow triggerRow = iterator.next();
            try {
                ScheduleId scheduleId = rowToScheduleId(triggerRow);
                if (schedulesFound.containsKey(scheduleId)) {
                    continue;
                }
                Optional<StructuredRow> optional = scheduleStore.read(getScheduleKeys(scheduleId));
                if (!optional.isPresent()) {
                    throw new NotFoundException(scheduleId);
                }
                StructuredRow scheduleRow = optional.get();
                String serialized = scheduleRow.getString(StoreDefinition.ProgramScheduleStore.SCHEDULE);
                if (serialized == null) {
                    throw new NotFoundException(scheduleId);
                }
                ProgramSchedule schedule = GSON.fromJson(serialized, ProgramSchedule.class);
                ProgramScheduleMeta meta = extractMetaFromRow(scheduleId, scheduleRow);
                ProgramScheduleRecord record = new ProgramScheduleRecord(schedule, meta);
                schedulesFound.put(scheduleId, record);
            } catch (IllegalArgumentException | NotFoundException e) {
                // the only exceptions we know to be thrown here are IllegalArgumentException (ill-formed key) or
                // NotFoundException (if the schedule does not exist). Both should never happen, so we warn and ignore.
                // we will let any other exception propagate up, because it would be a DataSetException or similarly serious.
                LOG.warn("Problem with trigger '{}' found for trigger key '{}': {}. Skipping entry.", triggerRow, triggerKey, e.getMessage());
            }
        }
    }
    return schedulesFound.values();
}

Example 5 with ProgramScheduleMeta

use of io.cdap.cdap.internal.app.runtime.schedule.ProgramScheduleMeta in project cdap by caskdata.

the class ProgramScheduleStoreDataset method getScheduleRecord.

/**
 * Read all information about a schedule from the store.
 *
 * @param scheduleId the id of the schedule to read
 * @return the schedule record from the store
 * @throws NotFoundException if the schedule does not exist in the store
 */
public ProgramScheduleRecord getScheduleRecord(ScheduleId scheduleId) throws NotFoundException, IOException {
    StructuredRow row = readExistingScheduleRow(scheduleId);
    String serializedSchedule = row.getString(StoreDefinition.ProgramScheduleStore.SCHEDULE);
    if (serializedSchedule == null) {
        throw new NotFoundException(scheduleId);
    }
    ProgramSchedule schedule = GSON.fromJson(serializedSchedule, ProgramSchedule.class);
    ProgramScheduleMeta meta = extractMetaFromRow(scheduleId, row);
    return new ProgramScheduleRecord(schedule, meta);
}