Examples with JobLifecycleListener org.apache.gobblin.runtime.api.JobLifecycleListener used on opensource projects

Example 1 with JobLifecycleListener

use of org.apache.gobblin.runtime.api.JobLifecycleListener in project incubator-gobblin by apache.

the class JobBrokerInjectionTest method testBrokerIsAcquiredAndShared.

@Test
public void testBrokerIsAcquiredAndShared() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitToJobCatalog");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("brokerTest/SimpleHelloWorldJob.jobconf")).build();
    final String eventBusId = js1.getConfig().resolve().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(100000, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    Queue<TestingEventBuses.Event> events = asserter.getEvents();
    Set<Long> seenInstanceObjectIds = Sets.newHashSet();
    Set<Long> seenJobObjectIds = Sets.newHashSet();
    Set<Long> seenTaskObjectIds = Sets.newHashSet();
    for (TestingEventBuses.Event event : events) {
        MyRecord record = (MyRecord) event.getValue();
        seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
        seenJobObjectIds.add(record.getJobSharedObjectId());
        seenTaskObjectIds.add(record.getTaskSharedObjectId());
    }
    // Should see same instance and job id (only 1 id in the set), but 5 different task ids for each task
    Assert.assertEquals(seenInstanceObjectIds.size(), 1);
    Assert.assertEquals(seenJobObjectIds.size(), 1);
    Assert.assertEquals(seenTaskObjectIds.size(), 5);
    asserter.clear();
    instance.getMutableJobCatalog().remove(js1.getUri());
    instance.getMutableJobCatalog().put(js1);
    jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    jobResult = jobDriver.get(10, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    events = asserter.getEvents();
    for (TestingEventBuses.Event event : events) {
        MyRecord record = (MyRecord) event.getValue();
        seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
        seenJobObjectIds.add(record.getJobSharedObjectId());
        seenTaskObjectIds.add(record.getTaskSharedObjectId());
    }
    // A different job should produce a new shared object id
    Assert.assertEquals(seenInstanceObjectIds.size(), 1);
    Assert.assertEquals(seenJobObjectIds.size(), 2);
    Assert.assertEquals(seenTaskObjectIds.size(), 10);
}

Example 2 with JobLifecycleListener

use of org.apache.gobblin.runtime.api.JobLifecycleListener in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method testDirectToScheduler.

@Test
public /**
 * Test running of a job when submitted directly to the scheduler
 */
void testDirectToScheduler() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testDirectToScheduler");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("gobblin/runtime/instance/SimpleHelloWorldJob.jobconf")).build();
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    JobSpecRunnable js1Runnable = instance.createJobSpecRunnable(js1);
    instance.getJobScheduler().scheduleOnce(js1, js1Runnable);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    instanceLauncher.stopAsync();
    instanceLauncher.awaitTerminated(5, TimeUnit.SECONDS);
}

Example 3 with JobLifecycleListener

use of org.apache.gobblin.runtime.api.JobLifecycleListener in project incubator-gobblin by apache.

the class TestJobLifecycleListenersList method testHappyPath.

@Test
public void testHappyPath() {
    Logger log = LoggerFactory.getLogger("testHappyPath");
    JobCatalogListenersContainer jobCatalog = mock(JobCatalogListenersContainer.class);
    JobSpecSchedulerListenersContainer jobScheduler = mock(JobSpecSchedulerListenersContainer.class);
    JobExecutionDriver mockDriver = mock(JobExecutionDriver.class);
    JobExecutionState mockState = mock(JobExecutionState.class);
    JobLifecycleListener listener1 = mock(JobLifecycleListener.class);
    JobLifecycleListener listener2 = mock(JobLifecycleListener.class);
    JobLifecycleListenersList disp = new JobLifecycleListenersList(jobCatalog, jobScheduler, log);
    disp.registerJobLifecycleListener(listener1);
    disp.onJobLaunch(mockDriver);
    disp.registerWeakJobLifecycleListener(listener2);
    disp.onMetadataChange(mockState, "key", "oldValue", "newValue");
    verify(jobCatalog).addListener(eq(listener1));
    verify(jobScheduler).registerJobSpecSchedulerListener(eq(listener1));
    verify(listener1).onJobLaunch(eq(mockDriver));
    verify(listener2, never()).onJobLaunch(eq(mockDriver));
    verify(jobCatalog).registerWeakJobCatalogListener(eq(listener2));
    verify(jobScheduler).registerWeakJobSpecSchedulerListener(eq(listener2));
    verify(listener1).onMetadataChange(eq(mockState), eq("key"), eq("oldValue"), eq("newValue"));
    verify(listener2).onMetadataChange(eq(mockState), eq("key"), eq("oldValue"), eq("newValue"));
}

Example 4 with JobLifecycleListener

use of org.apache.gobblin.runtime.api.JobLifecycleListener in project incubator-gobblin by apache.

the class TestFilteredJobLifecycleListener method testSimple.

@Test
public void testSimple() {
    Config config = ConfigFactory.empty().withValue(ConfigurationKeys.JOB_NAME_KEY, ConfigValueFactory.fromAnyRef("myJob"));
    JobSpec js1_1 = JobSpec.builder("gobblin:/testSimple/job1").withVersion("1").withConfig(config).build();
    JobSpec js1_2 = JobSpec.builder("gobblin:/testSimple/job1").withVersion("2").withConfig(config).build();
    JobLifecycleListener mockListener = mock(JobLifecycleListener.class);
    FilteredJobLifecycleListener testListener = new FilteredJobLifecycleListener(JobSpecFilter.builder().eqURI("gobblin:/testSimple/job1").eqVersion("2").build(), mockListener);
    JobExecutionState jss1_1 = new JobExecutionState(js1_1, JobExecutionUpdatable.createFromJobSpec(js1_1), Optional.<JobExecutionStateListener>absent());
    JobExecutionState jss1_2 = new JobExecutionState(js1_2, JobExecutionUpdatable.createFromJobSpec(js1_2), Optional.<JobExecutionStateListener>absent());
    testListener.onAddJob(js1_1);
    testListener.onDeleteJob(js1_1.getUri(), js1_1.getVersion());
    testListener.onUpdateJob(js1_1);
    ;
    testListener.onStatusChange(jss1_1, RunningState.PENDING, RunningState.RUNNING);
    testListener.onStageTransition(jss1_1, "Stage1", "Stage2");
    testListener.onMetadataChange(jss1_1, "metaKey", "value1", "value2");
    testListener.onAddJob(js1_2);
    testListener.onDeleteJob(js1_2.getUri(), js1_2.getVersion());
    testListener.onUpdateJob(js1_2);
    testListener.onStatusChange(jss1_2, RunningState.RUNNING, RunningState.SUCCESSFUL);
    testListener.onStageTransition(jss1_2, "Stage1", "Stage2");
    testListener.onMetadataChange(jss1_2, "metaKey", "value1", "value2");
    verify(mockListener).onAddJob(eq(js1_2));
    verify(mockListener).onDeleteJob(eq(js1_2.getUri()), eq(js1_2.getVersion()));
    verify(mockListener).onUpdateJob(eq(js1_2));
    verify(mockListener).onStatusChange(eq(jss1_2), eq(RunningState.RUNNING), eq(RunningState.SUCCESSFUL));
    verify(mockListener).onStageTransition(eq(jss1_2), eq("Stage1"), eq("Stage2"));
    verify(mockListener).onMetadataChange(eq(jss1_2), eq("metaKey"), eq("value1"), eq("value2"));
    verify(mockListener, never()).onAddJob(eq(js1_1));
    verify(mockListener, never()).onDeleteJob(eq(js1_1.getUri()), eq(js1_1.getVersion()));
    verify(mockListener, never()).onUpdateJob(eq(js1_1));
    verify(mockListener, never()).onStatusChange(eq(jss1_1), eq(RunningState.RUNNING), eq(RunningState.SUCCESSFUL));
    verify(mockListener, never()).onStatusChange(eq(jss1_1), eq(RunningState.PENDING), eq(RunningState.RUNNING));
    verify(mockListener, never()).onStageTransition(eq(jss1_1), eq("Stage1"), eq("Stage2"));
    verify(mockListener, never()).onMetadataChange(eq(jss1_1), eq("metaKey"), eq("value1"), eq("value2"));
}

Example 5 with JobLifecycleListener

use of org.apache.gobblin.runtime.api.JobLifecycleListener in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method testSubmitToJobCatalog.

@Test
public /**
 * Test running of a job using the standard path of submitting to the job catalog
 */
void testSubmitToJobCatalog() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitToJobCatalog");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("gobblin/runtime/instance/SimpleHelloWorldJob.jobconf")).build();
    final String eventBusId = js1.getConfig().resolve().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    instanceLauncher.stopAsync();
    final int numHellos = js1.getConfig().getInt(HelloWorldSource.NUM_HELLOS_FULL_KEY);
    ArrayList<String> expectedEvents = new ArrayList<>();
    for (int i = 1; i <= numHellos; ++i) {
        expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
    }
    asserter.assertNextValuesEq(expectedEvents);
    asserter.close();
    instanceLauncher.awaitTerminated(5, TimeUnit.SECONDS);
}