Examples with JobSpecSchedule org.apache.gobblin.runtime.api.JobSpecSchedule used on opensource projects

Example 1 with JobSpecSchedule

use of org.apache.gobblin.runtime.api.JobSpecSchedule in project incubator-gobblin by apache.

the class AbstractJobSpecScheduler method scheduleJob.

/**
 * {@inheritDoc}
 */
@Override
public JobSpecSchedule scheduleJob(JobSpec jobSpec, Runnable jobRunnable) {
    _log.info("Scheduling JobSpec " + jobSpec);
    final URI jobSpecURI = jobSpec.getUri();
    JobSpecSchedule newSchedule = null;
    Runnable runnableWithTriggerCallback = new TriggerRunnable(jobSpec, jobRunnable);
    synchronized (this) {
        JobSpecSchedule existingSchedule = _schedules.get(jobSpecURI);
        if (null != existingSchedule) {
            if (existingSchedule.getJobSpec().equals(jobSpec)) {
                _log.warn("Ignoring already scheduled job: " + jobSpec);
                return existingSchedule;
            }
            // a new job spec -- unschedule first so that we schedule the new version
            unscheduleJob(jobSpecURI);
        }
        newSchedule = doScheduleJob(jobSpec, runnableWithTriggerCallback);
        _schedules.put(jobSpecURI, newSchedule);
    }
    _callbacksDispatcher.onJobScheduled(newSchedule);
    return newSchedule;
}

Example 2 with JobSpecSchedule

use of org.apache.gobblin.runtime.api.JobSpecSchedule in project incubator-gobblin by apache.

the class TestQuartzJobSpecScheduler method testSchedule.

@Test
public void testSchedule() throws Exception {
    final Logger log = LoggerFactory.getLogger(getClass().getName() + ".testSchedule");
    Config quartzCfg = ConfigFactory.parseMap(ImmutableMap.<String, Object>builder().put("org.quartz.scheduler.instanceName", "TestQuartzJobSpecScheduler.testSchedule").put("org.quartz.threadPool.threadCount", "10").put("org.quartz.jobStore.class", "org.quartz.simpl.RAMJobStore").build());
    QuartzJobSpecScheduler scheduler = new QuartzJobSpecScheduler(log, quartzCfg);
    scheduler.startAsync();
    scheduler.awaitRunning(10, TimeUnit.SECONDS);
    Assert.assertTrue(scheduler._scheduler.getScheduler().isStarted());
    final ArrayBlockingQueue<JobSpec> expectedCalls = new ArrayBlockingQueue<>(100);
    try {
        Config jobCfg1 = ConfigFactory.parseMap(ImmutableMap.<String, Object>builder().put(ConfigurationKeys.JOB_SCHEDULE_KEY, "0/5 * * * * ?").build());
        Config jobCfg2 = ConfigFactory.parseMap(ImmutableMap.<String, Object>builder().put(ConfigurationKeys.JOB_SCHEDULE_KEY, "3/5 * * * * ?").build());
        final JobSpec js1 = JobSpec.builder("test.job1").withConfig(jobCfg1).build();
        final JobSpec js2 = JobSpec.builder("test.job2").withConfig(jobCfg2).build();
        final JobSpec js1_2 = JobSpec.builder("test.job1").withConfig(jobCfg1).withVersion("2").build();
        JobSpecSchedule jss1 = scheduler.scheduleJob(js1, new Runnable() {

            @Override
            public void run() {
                expectedCalls.offer(js1);
            }
        });
        Assert.assertEquals(scheduler.getSchedules().size(), 1);
        Assert.assertEquals(jss1.getJobSpec(), js1);
        Assert.assertTrue(jss1 instanceof QuartzJobSchedule);
        QuartzJobSchedule qjss1 = (QuartzJobSchedule) jss1;
        Assert.assertNotNull(scheduler._scheduler.getScheduler().getJobDetail(qjss1.getQuartzTrigger().getJobKey()));
        Assert.assertNotNull(scheduler._scheduler.getScheduler().getTrigger(qjss1.getQuartzTrigger().getKey()));
        Assert.assertTrue(qjss1.getQuartzTrigger().mayFireAgain());
        // Wait for the next run
        JobSpec expJs1 = expectedCalls.poll(6000, TimeUnit.MILLISECONDS);
        Assert.assertEquals(expJs1, js1);
        // Wait for the next run
        expJs1 = expectedCalls.poll(6000, TimeUnit.MILLISECONDS);
        Assert.assertEquals(expJs1, js1);
        // Schedule another job
        JobSpecSchedule jss2 = scheduler.scheduleJob(js2, new Runnable() {

            @Override
            public void run() {
                expectedCalls.offer(js2);
            }
        });
        Assert.assertEquals(scheduler.getSchedules().size(), 2);
        Assert.assertEquals(jss2.getJobSpec(), js2);
        // Wait for the next run -- we should get js2
        JobSpec expJs2 = expectedCalls.poll(6000, TimeUnit.MILLISECONDS);
        Assert.assertEquals(expJs2, js2);
        // Wait for the next run -- we should get js1
        expJs1 = expectedCalls.poll(6000, TimeUnit.MILLISECONDS);
        Assert.assertEquals(expJs1, js1);
        // Wait for the next run -- we should get js2
        expJs2 = expectedCalls.poll(6000, TimeUnit.MILLISECONDS);
        log.info("Found call: " + expJs2);
        Assert.assertEquals(expJs2, js2);
        // Update the first job
        QuartzJobSchedule qjss1_2 = (QuartzJobSchedule) scheduler.scheduleJob(js1_2, new Runnable() {

            @Override
            public void run() {
                expectedCalls.offer(js1_2);
            }
        });
        Assert.assertEquals(scheduler.getSchedules().size(), 2);
        // Wait for 5 seconds -- we should see at least 2 runs of js1_2 and js2
        Thread.sleep(15000);
        int js1_2_cnt = 0;
        int js2_cnt = 0;
        for (JobSpec nextJs : expectedCalls) {
            log.info("Found call: " + nextJs);
            if (js1_2.equals(nextJs)) {
                ++js1_2_cnt;
            } else if (js2.equals(nextJs)) {
                ++js2_cnt;
            } else {
                Assert.fail("Unexpected job spec: " + nextJs);
            }
        }
        Assert.assertTrue(js1_2_cnt >= 2, "js1_2_cnt=" + js1_2_cnt);
        Assert.assertTrue(js2_cnt >= 2, "js2_cnt=" + js2_cnt);
        scheduler.unscheduleJob(js1_2.getUri());
        Assert.assertEquals(scheduler.getSchedules().size(), 1);
        Assert.assertFalse(scheduler._scheduler.getScheduler().checkExists(qjss1_2.getQuartzTrigger().getJobKey()));
        // Flush calls
        Thread.sleep(1000);
        expectedCalls.clear();
        // All subsequent calls should be for js2
        for (int i = 0; i < 2; ++i) {
            JobSpec nextJs = expectedCalls.poll(12000, TimeUnit.MILLISECONDS);
            Assert.assertEquals(nextJs, js2);
        }
    } finally {
        scheduler.stopAsync();
        scheduler.awaitTerminated(10, TimeUnit.SECONDS);
    }
    // we may have to drain at most one call due to race conditions
    if (null != expectedCalls.poll(2100, TimeUnit.MILLISECONDS)) {
        Assert.assertNull(expectedCalls.poll(3000, TimeUnit.MILLISECONDS));
    }
}

Example 3 with JobSpecSchedule

use of org.apache.gobblin.runtime.api.JobSpecSchedule in project incubator-gobblin by apache.

the class TestImmediateJobSpecScheduler method testSchedule.

@Test
public void testSchedule() throws Exception {
    final Logger log = LoggerFactory.getLogger(getClass().getName() + ".testSimpleFlow");
    final Optional<Logger> logOpt = Optional.of(log);
    ImmediateJobSpecScheduler scheduler = new ImmediateJobSpecScheduler(logOpt);
    JobSpecSchedulerListener listener = mock(JobSpecSchedulerListener.class);
    scheduler.registerWeakJobSpecSchedulerListener(listener);
    final CountDownLatch expectedCallCount = new CountDownLatch(4);
    Runnable r = new Runnable() {

        @Override
        public void run() {
            expectedCallCount.countDown();
        }
    };
    JobSpec js1 = JobSpec.builder("test.job1").build();
    JobSpec js2 = JobSpec.builder("test.job2").build();
    JobSpec js3 = JobSpec.builder("test.job3").build();
    JobSpec js1_2 = JobSpec.builder("test.job1").withVersion("2").build();
    JobSpecSchedule jss1 = scheduler.scheduleJob(js1, r);
    Assert.assertEquals(scheduler.getSchedules().size(), 1);
    Assert.assertEquals(jss1.getJobSpec(), js1);
    Assert.assertTrue(jss1.getNextRunTimeMillis().isPresent());
    Assert.assertTrue(jss1.getNextRunTimeMillis().get().longValue() <= System.currentTimeMillis());
    ;
    JobSpecSchedule jss2 = scheduler.scheduleJob(js2, r);
    Assert.assertEquals(scheduler.getSchedules().size(), 2);
    Assert.assertEquals(jss2.getJobSpec(), js2);
    Assert.assertTrue(jss2.getNextRunTimeMillis().isPresent());
    Assert.assertTrue(jss2.getNextRunTimeMillis().get().longValue() <= System.currentTimeMillis());
    ;
    JobSpecSchedule jss3 = scheduler.scheduleJob(js3, r);
    Assert.assertEquals(scheduler.getSchedules().size(), 3);
    Assert.assertEquals(jss3.getJobSpec(), js3);
    JobSpecSchedule jss1_2 = scheduler.scheduleJob(js1_2, r);
    Assert.assertEquals(scheduler.getSchedules().size(), 3);
    Assert.assertEquals(jss1_2.getJobSpec(), js1_2);
    Assert.assertTrue(expectedCallCount.await(100, TimeUnit.MILLISECONDS));
    scheduler.unscheduleJob(js1.getUri());
    Assert.assertEquals(scheduler.getSchedules().size(), 2);
    scheduler.unscheduleJob(js1.getUri());
    Assert.assertEquals(scheduler.getSchedules().size(), 2);
    verify(listener).onJobScheduled(Mockito.eq(jss1));
    verify(listener).onJobTriggered(Mockito.eq(js1));
    verify(listener).onJobScheduled(Mockito.eq(jss2));
    verify(listener).onJobTriggered(Mockito.eq(js2));
    verify(listener).onJobScheduled(Mockito.eq(jss3));
    verify(listener).onJobTriggered(Mockito.eq(js3));
    verify(listener).onJobUnscheduled(Mockito.eq(jss1));
    verify(listener).onJobScheduled(Mockito.eq(jss1_2));
    verify(listener).onJobTriggered(Mockito.eq(js1_2));
    verify(listener, Mockito.times(1)).onJobUnscheduled(Mockito.eq(jss1_2));
}

Example 4 with JobSpecSchedule

use of org.apache.gobblin.runtime.api.JobSpecSchedule in project incubator-gobblin by apache.

the class ImmediateJobSpecScheduler method doScheduleJob.

@Override
protected JobSpecSchedule doScheduleJob(JobSpec jobSpec, Runnable jobRunnable) {
    Thread runThread = _jobRunnablesThreadFactory.newThread(jobRunnable);
    getLog().info("Starting JobSpec " + jobSpec + " in thread " + runThread.getName());
    JobSpecSchedule schedule = DefaultJobSpecScheduleImpl.createImmediateSchedule(jobSpec, jobRunnable);
    runThread.start();
    return schedule;
}

Example 5 with JobSpecSchedule

use of org.apache.gobblin.runtime.api.JobSpecSchedule in project incubator-gobblin by apache.

the class AbstractJobSpecScheduler method unscheduleJob.

/**
 * {@inheritDoc}
 */
@Override
public void unscheduleJob(URI jobSpecURI) {
    JobSpecSchedule existingSchedule = null;
    synchronized (this) {
        existingSchedule = _schedules.get(jobSpecURI);
        if (null != existingSchedule) {
            _log.info("Unscheduling " + existingSchedule);
            this._schedules.remove(jobSpecURI);
            doUnschedule(existingSchedule);
        }
    }
    if (null != existingSchedule) {
        _callbacksDispatcher.onJobUnscheduled(existingSchedule);
    }
}