Examples with ProgramSchedule co.cask.cdap.internal.app.runtime.schedule.ProgramSchedule used on opensource projects

Example 31 with ProgramSchedule

use of co.cask.cdap.internal.app.runtime.schedule.ProgramSchedule in project cdap by caskdata.

the class ProgramScheduleStoreDatasetTest method testFindSchedulesByEventAndUpdateSchedule.

@Test
public void testFindSchedulesByEventAndUpdateSchedule() throws Exception {
    DatasetFramework dsFramework = getInjector().getInstance(DatasetFramework.class);
    TransactionSystemClient txClient = getInjector().getInstance(TransactionSystemClient.class);
    TransactionExecutorFactory txExecutorFactory = new DynamicTransactionExecutorFactory(txClient);
    dsFramework.truncateInstance(Schedulers.STORE_DATASET_ID);
    final ProgramScheduleStoreDataset store = dsFramework.getDataset(Schedulers.STORE_DATASET_ID, new HashMap<String, String>(), null);
    Assert.assertNotNull(store);
    TransactionExecutor txExecutor = txExecutorFactory.createExecutor(Collections.singleton((TransactionAware) store));
    final ProgramSchedule sched11 = new ProgramSchedule("sched11", "one partition schedule", PROG1_ID, ImmutableMap.of("prop3", "abc"), new PartitionTrigger(DS1_ID, 1), ImmutableList.<Constraint>of());
    final ProgramSchedule sched12 = new ProgramSchedule("sched12", "two partition schedule", PROG1_ID, ImmutableMap.of("propper", "popper"), new PartitionTrigger(DS2_ID, 2), ImmutableList.<Constraint>of());
    final ProgramSchedule sched22 = new ProgramSchedule("sched22", "twentytwo partition schedule", PROG2_ID, ImmutableMap.of("nn", "4"), new PartitionTrigger(DS2_ID, 22), ImmutableList.<Constraint>of());
    final ProgramSchedule sched31 = new ProgramSchedule("sched31", "a program status trigger", PROG3_ID, ImmutableMap.of("propper", "popper"), new ProgramStatusTrigger(PROG1_ID, ProgramStatus.COMPLETED, ProgramStatus.FAILED, ProgramStatus.KILLED), ImmutableList.<Constraint>of());
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // event for DS1 or DS2 should trigger nothing. validate it returns an empty collection
            Assert.assertTrue(store.findSchedules(Schedulers.triggerKeyForPartition(DS1_ID)).isEmpty());
            Assert.assertTrue(store.findSchedules(Schedulers.triggerKeyForPartition(DS2_ID)).isEmpty());
            // event for PROG1 should trigger nothing. it should also return an empty collection
            Assert.assertTrue(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.COMPLETED)).isEmpty());
            Assert.assertTrue(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.FAILED)).isEmpty());
            Assert.assertTrue(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.KILLED)).isEmpty());
        }
    });
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            store.addSchedules(ImmutableList.of(sched11, sched12, sched22, sched31));
        }
    });
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // event for ProgramStatus should trigger only sched31
            Assert.assertEquals(ImmutableSet.of(sched31), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.COMPLETED))));
            Assert.assertEquals(ImmutableSet.of(sched31), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.FAILED))));
            Assert.assertEquals(ImmutableSet.of(sched31), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.KILLED))));
            // event for DS1 should trigger only sched11
            Assert.assertEquals(ImmutableSet.of(sched11), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForPartition(DS1_ID))));
            // event for DS2 triggers only sched12 and sched22
            Assert.assertEquals(ImmutableSet.of(sched12, sched22), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForPartition(DS2_ID))));
        }
    });
    final ProgramSchedule sched11New = new ProgramSchedule(sched11.getName(), "time schedule", PROG1_ID, ImmutableMap.of("timeprop", "time"), new TimeTrigger("* * * * *"), ImmutableList.<Constraint>of());
    final ProgramSchedule sched12New = new ProgramSchedule(sched12.getName(), "one partition schedule", PROG1_ID, ImmutableMap.of("pp", "p"), new PartitionTrigger(DS1_ID, 2), ImmutableList.<Constraint>of());
    final ProgramSchedule sched22New = new ProgramSchedule(sched22.getName(), "program3 failed schedule", PROG2_ID, ImmutableMap.of("ss", "s"), new ProgramStatusTrigger(PROG3_ID, ProgramStatus.FAILED), ImmutableList.<Constraint>of());
    final ProgramSchedule sched31New = new ProgramSchedule(sched31.getName(), "program1 failed schedule", PROG3_ID, ImmutableMap.of("abcd", "efgh"), new ProgramStatusTrigger(PROG1_ID, ProgramStatus.FAILED), ImmutableList.<Constraint>of());
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            store.updateSchedule(sched11New);
            store.updateSchedule(sched12New);
            store.updateSchedule(sched22New);
            store.updateSchedule(sched31New);
        }
    });
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // event for DS1 should trigger only sched12New after update
            Assert.assertEquals(ImmutableSet.of(sched12New), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForPartition(DS1_ID))));
            // event for DS2 triggers no schedule after update
            Assert.assertEquals(ImmutableSet.<ProgramSchedule>of(), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForPartition(DS2_ID))));
            // event for PS triggers only for failed program statuses, not completed nor killed
            Assert.assertEquals(ImmutableSet.of(sched31New), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.FAILED))));
            Assert.assertEquals(ImmutableSet.of(), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.COMPLETED))));
            Assert.assertEquals(ImmutableSet.of(), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.KILLED))));
        }
    });
}

Example 32 with ProgramSchedule

use of co.cask.cdap.internal.app.runtime.schedule.ProgramSchedule in project cdap by caskdata.

the class TriggerCodecTest method testContainingTrigger.

private void testContainingTrigger(ProtoTrigger proto, Trigger trigger) {
    ProgramSchedule proto1 = new ProgramSchedule("sched1", "one partition schedule", new NamespaceId("test").app("a").worker("ww"), ImmutableMap.of("prop3", "abc"), proto, ImmutableList.<Constraint>of());
    ProgramSchedule sched1 = new ProgramSchedule("sched1", "one partition schedule", new NamespaceId("test").app("a").worker("ww"), ImmutableMap.of("prop3", "abc"), trigger, ImmutableList.<Constraint>of());
    Assert.assertEquals(sched1, GSON.fromJson(GSON.toJson(proto1), ProgramSchedule.class));
}

Example 33 with ProgramSchedule

use of co.cask.cdap.internal.app.runtime.schedule.ProgramSchedule in project cdap by caskdata.

the class LastRunConstraintTest method testLastRunConstraint.

@Test
public void testLastRunConstraint() {
    Store store = AppFabricTestHelper.getInjector().getInstance(Store.class);
    long now = System.currentTimeMillis();
    long nowSec = TimeUnit.MILLISECONDS.toSeconds(now);
    ProgramSchedule schedule = new ProgramSchedule("SCHED1", "one partition schedule", WORKFLOW_ID, ImmutableMap.of("prop3", "abc"), new PartitionTrigger(DATASET_ID, 1), ImmutableList.of());
    SimpleJob job = new SimpleJob(schedule, now, Collections.emptyList(), Job.State.PENDING_TRIGGER, 0L);
    // require 1 hour since last run
    LastRunConstraint lastRunConstraint = new LastRunConstraint(1, TimeUnit.HOURS);
    ConstraintContext constraintContext = new ConstraintContext(job, now, store);
    // there's been no runs, so the constraint is satisfied by default
    assertSatisfied(true, lastRunConstraint.check(schedule, constraintContext));
    ProgramRunId pid1 = WORKFLOW_ID.run(RunIds.generate().getId());
    ProgramRunId pid2 = WORKFLOW_ID.run(RunIds.generate().getId());
    ProgramRunId pid3 = WORKFLOW_ID.run(RunIds.generate().getId());
    ProgramRunId pid4 = WORKFLOW_ID.run(RunIds.generate().getId());
    // a RUNNING workflow, started 3 hours ago will fail the constraint check
    Map<String, String> systemArgs = ImmutableMap.of(ProgramOptionConstants.SCHEDULE_NAME, schedule.getName());
    long startTime = nowSec - TimeUnit.HOURS.toSeconds(3);
    setStartAndRunning(store, pid1, startTime, EMPTY_MAP, systemArgs);
    assertSatisfied(false, lastRunConstraint.check(schedule, constraintContext));
    // a SUSPENDED workflow started 3 hours ago will also fail the constraint check
    store.setSuspend(pid1, AppFabricTestHelper.createSourceId(++sourceId));
    assertSatisfied(false, lastRunConstraint.check(schedule, constraintContext));
    store.setResume(pid1, AppFabricTestHelper.createSourceId(++sourceId));
    assertSatisfied(false, lastRunConstraint.check(schedule, constraintContext));
    // if that same workflow runs completes 2 hours ago, the constraint check will be satisfied
    store.setStop(pid1, nowSec - TimeUnit.HOURS.toSeconds(2), ProgramRunStatus.COMPLETED, AppFabricTestHelper.createSourceId(++sourceId));
    assertSatisfied(true, lastRunConstraint.check(schedule, constraintContext));
    // a RUNNING workflow, started 2 hours ago will fail the constraint check
    startTime = nowSec - TimeUnit.HOURS.toSeconds(2);
    setStartAndRunning(store, pid2, startTime);
    assertSatisfied(false, lastRunConstraint.check(schedule, constraintContext));
    // if that same workflow run fails 1 minute ago, the constraint check will be satisfied
    store.setStop(pid2, nowSec - TimeUnit.MINUTES.toSeconds(1), ProgramRunStatus.FAILED, AppFabricTestHelper.createSourceId(++sourceId));
    assertSatisfied(true, lastRunConstraint.check(schedule, constraintContext));
    // similarly, a KILLED workflow, started 2 hours ago will also fail the constraint check
    startTime = nowSec - TimeUnit.HOURS.toSeconds(2);
    setStartAndRunning(store, pid3, startTime);
    assertSatisfied(false, lastRunConstraint.check(schedule, constraintContext));
    store.setStop(pid3, nowSec - TimeUnit.MINUTES.toSeconds(1), ProgramRunStatus.KILLED, AppFabricTestHelper.createSourceId(++sourceId));
    assertSatisfied(true, lastRunConstraint.check(schedule, constraintContext));
    // a RUNNING workflow, started 2 hours ago will fail the constraint check
    startTime = nowSec - TimeUnit.HOURS.toSeconds(2);
    setStartAndRunning(store, pid4, startTime);
    assertSatisfied(false, lastRunConstraint.check(schedule, constraintContext));
    // if that same workflow runs completes 1 minute ago, the constraint check will not be satisfied
    store.setStop(pid4, nowSec - TimeUnit.MINUTES.toSeconds(1), ProgramRunStatus.COMPLETED, AppFabricTestHelper.createSourceId(++sourceId));
    assertSatisfied(false, lastRunConstraint.check(schedule, constraintContext));
}

Example 34 with ProgramSchedule

use of co.cask.cdap.internal.app.runtime.schedule.ProgramSchedule in project cdap by caskdata.

the class CoreSchedulerServiceTest method addListDeleteSchedules.

@Test
public void addListDeleteSchedules() throws Exception {
    // verify that list returns nothing
    Assert.assertTrue(scheduler.listSchedules(APP1_ID).isEmpty());
    Assert.assertTrue(scheduler.listSchedules(PROG1_ID).isEmpty());
    // add a schedule for app1
    ProgramSchedule tsched1 = new ProgramSchedule("tsched1", "one time schedule", PROG1_ID, ImmutableMap.of("prop1", "nn"), new TimeTrigger("* * ? * 1"), ImmutableList.<Constraint>of());
    scheduler.addSchedule(tsched1);
    Assert.assertEquals(tsched1, scheduler.getSchedule(TSCHED1_ID));
    Assert.assertEquals(ImmutableList.of(tsched1), scheduler.listSchedules(APP1_ID));
    Assert.assertEquals(ImmutableList.of(tsched1), scheduler.listSchedules(PROG1_ID));
    // add three more schedules, one for the same program, one for the same app, one for another app
    ProgramSchedule psched1 = new ProgramSchedule("psched1", "one partition schedule", PROG1_ID, ImmutableMap.of("prop3", "abc"), new PartitionTrigger(DS1_ID, 1), ImmutableList.<Constraint>of());
    ProgramSchedule tsched11 = new ProgramSchedule("tsched11", "two times schedule", PROG11_ID, ImmutableMap.of("prop2", "xx"), new TimeTrigger("* * ? * 1,2"), ImmutableList.<Constraint>of());
    ProgramSchedule psched2 = new ProgramSchedule("psched2", "two partition schedule", PROG2_ID, ImmutableMap.of("propper", "popper"), new PartitionTrigger(DS2_ID, 2), ImmutableList.<Constraint>of());
    scheduler.addSchedules(ImmutableList.of(psched1, tsched11, psched2));
    Assert.assertEquals(psched1, scheduler.getSchedule(PSCHED1_ID));
    Assert.assertEquals(tsched11, scheduler.getSchedule(TSCHED11_ID));
    Assert.assertEquals(psched2, scheduler.getSchedule(PSCHED2_ID));
    // list by app and program
    Assert.assertEquals(ImmutableList.of(psched1, tsched1), scheduler.listSchedules(PROG1_ID));
    Assert.assertEquals(ImmutableList.of(tsched11), scheduler.listSchedules(PROG11_ID));
    Assert.assertEquals(ImmutableList.of(psched2), scheduler.listSchedules(PROG2_ID));
    Assert.assertEquals(ImmutableList.of(psched1, tsched1, tsched11), scheduler.listSchedules(APP1_ID));
    Assert.assertEquals(ImmutableList.of(psched2), scheduler.listSchedules(APP2_ID));
    // delete one schedule
    scheduler.deleteSchedule(TSCHED1_ID);
    verifyNotFound(scheduler, TSCHED1_ID);
    Assert.assertEquals(ImmutableList.of(psched1), scheduler.listSchedules(PROG1_ID));
    Assert.assertEquals(ImmutableList.of(tsched11), scheduler.listSchedules(PROG11_ID));
    Assert.assertEquals(ImmutableList.of(psched2), scheduler.listSchedules(PROG2_ID));
    Assert.assertEquals(ImmutableList.of(psched1, tsched11), scheduler.listSchedules(APP1_ID));
    Assert.assertEquals(ImmutableList.of(psched2), scheduler.listSchedules(APP2_ID));
    // attempt to delete it again along with another one that exists
    try {
        scheduler.deleteSchedules(ImmutableList.of(TSCHED1_ID, TSCHED11_ID));
        Assert.fail("expected NotFoundException");
    } catch (NotFoundException e) {
    // expected
    }
    Assert.assertEquals(ImmutableList.of(psched1), scheduler.listSchedules(PROG1_ID));
    Assert.assertEquals(ImmutableList.of(tsched11), scheduler.listSchedules(PROG11_ID));
    Assert.assertEquals(ImmutableList.of(psched2), scheduler.listSchedules(PROG2_ID));
    Assert.assertEquals(ImmutableList.of(psched1, tsched11), scheduler.listSchedules(APP1_ID));
    Assert.assertEquals(ImmutableList.of(psched2), scheduler.listSchedules(APP2_ID));
    // attempt to add it back together with a schedule that exists
    try {
        scheduler.addSchedules(ImmutableList.of(tsched1, tsched11));
        Assert.fail("expected AlreadyExistsException");
    } catch (AlreadyExistsException e) {
    // expected
    }
    Assert.assertEquals(ImmutableList.of(psched1), scheduler.listSchedules(PROG1_ID));
    Assert.assertEquals(ImmutableList.of(tsched11), scheduler.listSchedules(PROG11_ID));
    Assert.assertEquals(ImmutableList.of(psched2), scheduler.listSchedules(PROG2_ID));
    Assert.assertEquals(ImmutableList.of(psched1, tsched11), scheduler.listSchedules(APP1_ID));
    Assert.assertEquals(ImmutableList.of(psched2), scheduler.listSchedules(APP2_ID));
    // add it back, delete all schedules for one app
    scheduler.addSchedule(tsched1);
    scheduler.deleteSchedules(APP1_ID);
    verifyNotFound(scheduler, TSCHED1_ID);
    verifyNotFound(scheduler, PSCHED1_ID);
    verifyNotFound(scheduler, TSCHED11_ID);
    Assert.assertEquals(ImmutableList.of(), scheduler.listSchedules(PROG1_ID));
    Assert.assertEquals(ImmutableList.of(), scheduler.listSchedules(PROG11_ID));
    Assert.assertEquals(ImmutableList.of(psched2), scheduler.listSchedules(PROG2_ID));
    Assert.assertEquals(ImmutableList.of(), scheduler.listSchedules(APP1_ID));
    Assert.assertEquals(ImmutableList.of(psched2), scheduler.listSchedules(PROG2_ID));
}

Example 35 with ProgramSchedule

use of co.cask.cdap.internal.app.runtime.schedule.ProgramSchedule in project cdap by caskdata.

the class CoreSchedulerServiceTest method testScheduleUpdate.

private void testScheduleUpdate(String howToUpdate) throws Exception {
    int runs = getRuns(WORKFLOW_2, ProgramRunStatus.ALL);
    final ScheduleId scheduleId2 = APP_ID.schedule(AppWithFrequentScheduledWorkflows.DATASET_PARTITION_SCHEDULE_2);
    // send one notification to it
    long minPublishTime = System.currentTimeMillis();
    publishNotification(dataEventTopic, NamespaceId.DEFAULT, AppWithFrequentScheduledWorkflows.DATASET_NAME2);
    waitUntilProcessed(dataEventTopic, minPublishTime);
    // A pending job will be created, but it won't run
    Assert.assertTrue("Expected a PENDING_TRIGGER job for " + scheduleId2, Iterables.any(getAllJobs(), new Predicate<Job>() {

        @Override
        public boolean apply(Job job) {
            if (!(job.getSchedule().getTrigger() instanceof ProtoTrigger.PartitionTrigger)) {
                return false;
            }
            return scheduleId2.equals(job.getJobKey().getScheduleId()) && job.getState() == Job.State.PENDING_TRIGGER;
        }
    }));
    Assert.assertEquals(runs, getRuns(WORKFLOW_2, ProgramRunStatus.ALL));
    if ("disable".equals(howToUpdate)) {
        // disabling and enabling the schedule should remove the job
        disableSchedule(AppWithFrequentScheduledWorkflows.DATASET_PARTITION_SCHEDULE_2);
        enableSchedule(AppWithFrequentScheduledWorkflows.DATASET_PARTITION_SCHEDULE_2);
    } else {
        ProgramSchedule schedule = scheduler.getSchedule(scheduleId2);
        Map<String, String> updatedProperties = ImmutableMap.<String, String>builder().putAll(schedule.getProperties()).put(howToUpdate, howToUpdate).build();
        ProgramSchedule updatedSchedule = new ProgramSchedule(schedule.getName(), schedule.getDescription(), schedule.getProgramId(), updatedProperties, schedule.getTrigger(), schedule.getConstraints());
        if ("update".equals(howToUpdate)) {
            scheduler.updateSchedule(updatedSchedule);
            Assert.assertEquals(ProgramScheduleStatus.SCHEDULED, scheduler.getScheduleStatus(scheduleId2));
        } else if ("delete".equals(howToUpdate)) {
            scheduler.deleteSchedule(scheduleId2);
            scheduler.addSchedule(updatedSchedule);
            enableSchedule(scheduleId2.getSchedule());
        } else {
            Assert.fail("invalid howToUpdate: " + howToUpdate);
        }
    }
    // single notification should not trigger workflow 2 yet (if it does, then the job was not removed)
    minPublishTime = System.currentTimeMillis();
    publishNotification(dataEventTopic, NamespaceId.DEFAULT, AppWithFrequentScheduledWorkflows.DATASET_NAME2);
    waitUntilProcessed(dataEventTopic, minPublishTime);
    // Again, a pending job will be created, but it won't run since updating the schedule would remove pending trigger
    Assert.assertTrue("Expected a PENDING_TRIGGER job for " + scheduleId2, Iterables.any(getAllJobs(), new Predicate<Job>() {

        @Override
        public boolean apply(Job job) {
            if (!(job.getSchedule().getTrigger() instanceof ProtoTrigger.PartitionTrigger)) {
                return false;
            }
            return scheduleId2.equals(job.getJobKey().getScheduleId()) && job.getState() == Job.State.PENDING_TRIGGER;
        }
    }));
    Assert.assertEquals(runs, getRuns(WORKFLOW_2, ProgramRunStatus.ALL));
    // publish one more notification, this should kick off the workflow
    publishNotification(dataEventTopic, NamespaceId.DEFAULT, AppWithFrequentScheduledWorkflows.DATASET_NAME2);
    waitForCompleteRuns(runs + 1, WORKFLOW_2);
}