Examples with JobSpec org.apache.gobblin.runtime.api.JobSpec used on opensource projects

Example 36 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class TestJobExecutionStateListeners method testHappyPath.

@Test
public void testHappyPath() {
    final Logger log = LoggerFactory.getLogger(getClass() + ".testHappyPath");
    JobExecutionStateListeners listeners = new JobExecutionStateListeners(log);
    JobSpec js1 = JobSpec.builder("gobblin:test/job").withConfig(ConfigFactory.empty().withValue(ConfigurationKeys.JOB_NAME_KEY, ConfigValueFactory.fromAnyRef("myJob"))).build();
    JobExecutionUpdatable je1 = JobExecutionUpdatable.createFromJobSpec(js1);
    JobExecutionStateListener l1 = Mockito.mock(JobExecutionStateListener.class);
    JobExecutionStateListener l2 = Mockito.mock(JobExecutionStateListener.class);
    JobExecutionStateListener l3 = Mockito.mock(JobExecutionStateListener.class);
    listeners.registerStateListener(l1);
    JobExecutionState state = new JobExecutionState(js1, je1, Optional.<JobExecutionStateListener>of(listeners));
    state.setRunningState(RunningState.PENDING);
    state.setRunningState(RunningState.RUNNING);
    listeners.registerStateListener(l2);
    listeners.registerStateListener(l3);
    state.setStage("Stage1");
    listeners.unregisterStateListener(l2);
    listeners.onMetadataChange(state, "key", "oldValue", "newValue");
    Mockito.verify(l1).onStatusChange(Mockito.eq(state), Mockito.eq((RunningState) null), Mockito.eq(RunningState.PENDING));
    Mockito.verify(l1).onStatusChange(Mockito.eq(state), Mockito.eq(RunningState.PENDING), Mockito.eq(RunningState.RUNNING));
    Mockito.verify(l1).onStageTransition(Mockito.eq(state), Mockito.eq(JobExecutionState.UKNOWN_STAGE), Mockito.eq("Stage1"));
    Mockito.verify(l1).onMetadataChange(Mockito.eq(state), Mockito.eq("key"), Mockito.eq("oldValue"), Mockito.eq("newValue"));
    Mockito.verify(l2).onStageTransition(Mockito.eq(state), Mockito.eq(JobExecutionState.UKNOWN_STAGE), Mockito.eq("Stage1"));
    Mockito.verify(l3).onStageTransition(Mockito.eq(state), Mockito.eq(JobExecutionState.UKNOWN_STAGE), Mockito.eq("Stage1"));
    Mockito.verify(l3).onMetadataChange(Mockito.eq(state), Mockito.eq("key"), Mockito.eq("oldValue"), Mockito.eq("newValue"));
    Mockito.verifyNoMoreInteractions(l1);
    Mockito.verifyNoMoreInteractions(l2);
    Mockito.verifyNoMoreInteractions(l3);
}

Example 37 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class TestJobSpecFilter method testUriAndVersion.

@Test
public void testUriAndVersion() {
    JobSpec js1_1 = JobSpec.builder("gobblin:/test/job1").withVersion("1").build();
    JobSpec js1_2 = JobSpec.builder("gobblin:/test/job1").withVersion("2").build();
    JobSpec js2_1 = JobSpec.builder("gobblin:/test/job2").withVersion("1").build();
    JobSpec js2_2 = JobSpec.builder("gobblin:/test/job2").withVersion("2").build();
    JobSpecFilter filter1 = JobSpecFilter.eqJobSpecURI("gobblin:/test/job1");
    Assert.assertTrue(filter1.apply(js1_1));
    Assert.assertTrue(filter1.apply(js1_2));
    Assert.assertFalse(filter1.apply(js2_1));
    Assert.assertFalse(filter1.apply(js2_2));
    JobSpecFilter filter2 = JobSpecFilter.builder().eqURI("gobblin:/test/job2").eqVersion("2").build();
    Assert.assertFalse(filter2.apply(js1_1));
    Assert.assertFalse(filter2.apply(js1_2));
    Assert.assertFalse(filter2.apply(js2_1));
    Assert.assertTrue(filter2.apply(js2_2));
}

Example 38 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class FileBasedJobLockFactoryManagerTest method testGetJobLockFactory.

@Test
public void testGetJobLockFactory() throws Exception {
    final Logger log = LoggerFactory.getLogger("FileBasedJobLockFactoryManagerTest.testGetJobLockFactory");
    FileBasedJobLockFactoryManager mgr = new FileBasedJobLockFactoryManager();
    // Create an instance with default configs
    Config sysConfig1 = ConfigFactory.empty();
    FileBasedJobLockFactory factory1 = mgr.getJobLockFactory(sysConfig1, Optional.of(log));
    Assert.assertTrue(factory1.getLockFileDir().toString().startsWith(FileBasedJobLockFactory.DEFAULT_LOCK_DIR_PREFIX));
    Assert.assertTrue(factory1.getFs() instanceof LocalFileSystem);
    Assert.assertTrue(factory1.getFs().exists(factory1.getLockFileDir()));
    JobSpec js1 = JobSpec.builder("gobblin-test:job1").build();
    FileBasedJobLock lock11 = factory1.getJobLock(js1);
    Assert.assertTrue(lock11.getLockFile().getName().startsWith(FileBasedJobLockFactory.getJobName(js1)));
    Assert.assertTrue(lock11.tryLock());
    lock11.unlock();
    // Lock dir should be deleted after close()
    factory1.close();
    Assert.assertFalse(factory1.getFs().exists(factory1.getLockFileDir()));
    // Create an instance with pre-existing lock dir
    File lockDir = Files.createTempDir();
    Assert.assertTrue(lockDir.exists());
    Config sysConfig2 = ConfigFactory.empty().withValue(FileBasedJobLockFactoryManager.CONFIG_PREFIX + "." + FileBasedJobLockFactory.LOCK_DIR_CONFIG, ConfigValueFactory.fromAnyRef(lockDir.getAbsolutePath()));
    FileBasedJobLockFactory factory2 = mgr.getJobLockFactory(sysConfig2, Optional.of(log));
    Assert.assertEquals(factory2.getLockFileDir().toString(), lockDir.getAbsolutePath());
    Assert.assertTrue(factory2.getFs() instanceof LocalFileSystem);
    Assert.assertTrue(factory2.getFs().exists(factory2.getLockFileDir()));
    // Lock dir should not be removed on close
    factory2.close();
    Assert.assertTrue(factory2.getFs().exists(factory2.getLockFileDir()));
}

Example 39 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class TestImmediateJobSpecScheduler method testSchedule.

@Test
public void testSchedule() throws Exception {
    final Logger log = LoggerFactory.getLogger(getClass().getName() + ".testSimpleFlow");
    final Optional<Logger> logOpt = Optional.of(log);
    ImmediateJobSpecScheduler scheduler = new ImmediateJobSpecScheduler(logOpt);
    JobSpecSchedulerListener listener = mock(JobSpecSchedulerListener.class);
    scheduler.registerWeakJobSpecSchedulerListener(listener);
    final CountDownLatch expectedCallCount = new CountDownLatch(4);
    Runnable r = new Runnable() {

        @Override
        public void run() {
            expectedCallCount.countDown();
        }
    };
    JobSpec js1 = JobSpec.builder("test.job1").build();
    JobSpec js2 = JobSpec.builder("test.job2").build();
    JobSpec js3 = JobSpec.builder("test.job3").build();
    JobSpec js1_2 = JobSpec.builder("test.job1").withVersion("2").build();
    JobSpecSchedule jss1 = scheduler.scheduleJob(js1, r);
    Assert.assertEquals(scheduler.getSchedules().size(), 1);
    Assert.assertEquals(jss1.getJobSpec(), js1);
    Assert.assertTrue(jss1.getNextRunTimeMillis().isPresent());
    Assert.assertTrue(jss1.getNextRunTimeMillis().get().longValue() <= System.currentTimeMillis());
    ;
    JobSpecSchedule jss2 = scheduler.scheduleJob(js2, r);
    Assert.assertEquals(scheduler.getSchedules().size(), 2);
    Assert.assertEquals(jss2.getJobSpec(), js2);
    Assert.assertTrue(jss2.getNextRunTimeMillis().isPresent());
    Assert.assertTrue(jss2.getNextRunTimeMillis().get().longValue() <= System.currentTimeMillis());
    ;
    JobSpecSchedule jss3 = scheduler.scheduleJob(js3, r);
    Assert.assertEquals(scheduler.getSchedules().size(), 3);
    Assert.assertEquals(jss3.getJobSpec(), js3);
    JobSpecSchedule jss1_2 = scheduler.scheduleJob(js1_2, r);
    Assert.assertEquals(scheduler.getSchedules().size(), 3);
    Assert.assertEquals(jss1_2.getJobSpec(), js1_2);
    Assert.assertTrue(expectedCallCount.await(100, TimeUnit.MILLISECONDS));
    scheduler.unscheduleJob(js1.getUri());
    Assert.assertEquals(scheduler.getSchedules().size(), 2);
    scheduler.unscheduleJob(js1.getUri());
    Assert.assertEquals(scheduler.getSchedules().size(), 2);
    verify(listener).onJobScheduled(Mockito.eq(jss1));
    verify(listener).onJobTriggered(Mockito.eq(js1));
    verify(listener).onJobScheduled(Mockito.eq(jss2));
    verify(listener).onJobTriggered(Mockito.eq(js2));
    verify(listener).onJobScheduled(Mockito.eq(jss3));
    verify(listener).onJobTriggered(Mockito.eq(js3));
    verify(listener).onJobUnscheduled(Mockito.eq(jss1));
    verify(listener).onJobScheduled(Mockito.eq(jss1_2));
    verify(listener).onJobTriggered(Mockito.eq(js1_2));
    verify(listener, Mockito.times(1)).onJobUnscheduled(Mockito.eq(jss1_2));
}

Example 40 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class TestDefaultGobblinInstanceDriverImpl method testScheduling.

@Test
public void testScheduling() throws Exception {
    final Logger log = LoggerFactory.getLogger(getClass().getName() + ".testScheduling");
    final Optional<Logger> loggerOpt = Optional.of(log);
    InMemoryJobCatalog jobCatalog = new InMemoryJobCatalog(loggerOpt);
    JobSpecScheduler scheduler = Mockito.mock(JobSpecScheduler.class);
    JobExecutionLauncher jobLauncher = Mockito.mock(JobExecutionLauncher.class);
    Configurable sysConfig = DefaultConfigurableImpl.createFromConfig(ConfigFactory.empty());
    final DefaultGobblinInstanceDriverImpl driver = new StandardGobblinInstanceDriver("testScheduling", sysConfig, jobCatalog, scheduler, jobLauncher, Optional.<MetricContext>absent(), loggerOpt, Collections.<GobblinInstancePluginFactory>emptyList(), SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()));
    JobSpec js1_1 = JobSpec.builder("test.job1").withVersion("1").build();
    JobSpec js1_2 = JobSpec.builder("test.job1").withVersion("2").build();
    JobSpec js2 = JobSpec.builder("test.job2").withVersion("1").build();
    driver.startAsync().awaitRunning(1000, TimeUnit.MILLISECONDS);
    long startTimeMs = System.currentTimeMillis();
    Assert.assertTrue(driver.isRunning());
    Assert.assertTrue(driver.isInstrumentationEnabled());
    Assert.assertNotNull(driver.getMetricContext());
    jobCatalog.put(js1_1);
    AssertWithBackoff awb = AssertWithBackoff.create().backoffFactor(1.5).maxSleepMs(100).timeoutMs(1000).logger(log);
    awb.assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            log.debug("upFlag=" + driver.getMetrics().getUpFlag().getValue().intValue());
            return driver.getMetrics().getUpFlag().getValue().intValue() == 1;
        }
    }, "upFlag==1");
    jobCatalog.put(js2);
    jobCatalog.put(js1_2);
    jobCatalog.remove(js2.getUri());
    Mockito.when(scheduler.scheduleJob(Mockito.eq(js1_1), Mockito.any(DefaultGobblinInstanceDriverImpl.JobSpecRunnable.class))).thenReturn(DefaultJobSpecScheduleImpl.createNoSchedule(js1_1, null));
    Mockito.when(scheduler.scheduleJob(Mockito.eq(js2), Mockito.any(DefaultGobblinInstanceDriverImpl.JobSpecRunnable.class))).thenReturn(DefaultJobSpecScheduleImpl.createNoSchedule(js2, null));
    Mockito.when(scheduler.scheduleJob(Mockito.eq(js1_2), Mockito.any(DefaultGobblinInstanceDriverImpl.JobSpecRunnable.class))).thenReturn(DefaultJobSpecScheduleImpl.createNoSchedule(js1_2, null));
    Mockito.verify(scheduler).scheduleJob(Mockito.eq(js1_1), Mockito.any(DefaultGobblinInstanceDriverImpl.JobSpecRunnable.class));
    Mockito.verify(scheduler).scheduleJob(Mockito.eq(js2), Mockito.any(DefaultGobblinInstanceDriverImpl.JobSpecRunnable.class));
    Mockito.verify(scheduler).scheduleJob(Mockito.eq(js1_2), Mockito.any(DefaultGobblinInstanceDriverImpl.JobSpecRunnable.class));
    Mockito.verify(scheduler).unscheduleJob(Mockito.eq(js2.getUri()));
    final long elapsedMs = System.currentTimeMillis() - startTimeMs;
    awb.assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            long uptimeMs = driver.getMetrics().getUptimeMs().getValue().longValue();
            return uptimeMs >= elapsedMs;
        }
    }, "uptime > elapsedMs");
    long uptimeMs = driver.getMetrics().getUptimeMs().getValue().longValue();
    Assert.assertTrue(uptimeMs <= 2 * elapsedMs, "uptime=" + uptimeMs + " elapsedMs=" + elapsedMs);
    driver.stopAsync();
    driver.awaitTerminated(100, TimeUnit.MILLISECONDS);
    Assert.assertEquals(driver.state(), State.TERMINATED);
    Assert.assertEquals(driver.getMetrics().getUpFlag().getValue().intValue(), 0);
    // Need an assert with retries because Guava service container notifications are async
    awb.assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            log.debug("upTimeMs=" + driver.getMetrics().getUptimeMs().getValue().longValue());
            return driver.getMetrics().getUptimeMs().getValue().longValue() == 0;
        }
    }, "upTimeMs==0");
}