Examples with JobExecutionDriver org.apache.gobblin.runtime.api.JobExecutionDriver used on opensource projects

Example 6 with JobExecutionDriver

use of org.apache.gobblin.runtime.api.JobExecutionDriver in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method testDirectToScheduler.

@Test
public /**
 * Test running of a job when submitted directly to the scheduler
 */
void testDirectToScheduler() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testDirectToScheduler");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("gobblin/runtime/instance/SimpleHelloWorldJob.jobconf")).build();
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    JobSpecRunnable js1Runnable = instance.createJobSpecRunnable(js1);
    instance.getJobScheduler().scheduleOnce(js1, js1Runnable);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    instanceLauncher.stopAsync();
    instanceLauncher.awaitTerminated(5, TimeUnit.SECONDS);
}

Example 7 with JobExecutionDriver

use of org.apache.gobblin.runtime.api.JobExecutionDriver in project incubator-gobblin by apache.

the class TestJobLifecycleListenersList method testHappyPath.

@Test
public void testHappyPath() {
    Logger log = LoggerFactory.getLogger("testHappyPath");
    JobCatalogListenersContainer jobCatalog = mock(JobCatalogListenersContainer.class);
    JobSpecSchedulerListenersContainer jobScheduler = mock(JobSpecSchedulerListenersContainer.class);
    JobExecutionDriver mockDriver = mock(JobExecutionDriver.class);
    JobExecutionState mockState = mock(JobExecutionState.class);
    JobLifecycleListener listener1 = mock(JobLifecycleListener.class);
    JobLifecycleListener listener2 = mock(JobLifecycleListener.class);
    JobLifecycleListenersList disp = new JobLifecycleListenersList(jobCatalog, jobScheduler, log);
    disp.registerJobLifecycleListener(listener1);
    disp.onJobLaunch(mockDriver);
    disp.registerWeakJobLifecycleListener(listener2);
    disp.onMetadataChange(mockState, "key", "oldValue", "newValue");
    verify(jobCatalog).addListener(eq(listener1));
    verify(jobScheduler).registerJobSpecSchedulerListener(eq(listener1));
    verify(listener1).onJobLaunch(eq(mockDriver));
    verify(listener2, never()).onJobLaunch(eq(mockDriver));
    verify(jobCatalog).registerWeakJobCatalogListener(eq(listener2));
    verify(jobScheduler).registerWeakJobSpecSchedulerListener(eq(listener2));
    verify(listener1).onMetadataChange(eq(mockState), eq("key"), eq("oldValue"), eq("newValue"));
    verify(listener2).onMetadataChange(eq(mockState), eq("key"), eq("oldValue"), eq("newValue"));
}

Example 8 with JobExecutionDriver

use of org.apache.gobblin.runtime.api.JobExecutionDriver in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method testSubmitToJobCatalog.

@Test
public /**
 * Test running of a job using the standard path of submitting to the job catalog
 */
void testSubmitToJobCatalog() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitToJobCatalog");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("gobblin/runtime/instance/SimpleHelloWorldJob.jobconf")).build();
    final String eventBusId = js1.getConfig().resolve().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    instanceLauncher.stopAsync();
    final int numHellos = js1.getConfig().getInt(HelloWorldSource.NUM_HELLOS_FULL_KEY);
    ArrayList<String> expectedEvents = new ArrayList<>();
    for (int i = 1; i <= numHellos; ++i) {
        expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
    }
    asserter.assertNextValuesEq(expectedEvents);
    asserter.close();
    instanceLauncher.awaitTerminated(5, TimeUnit.SECONDS);
}

Example 9 with JobExecutionDriver

use of org.apache.gobblin.runtime.api.JobExecutionDriver in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method testSubmitWithTemplate.

@Test
public void testSubmitWithTemplate() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitWithTemplate");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseMap(ImmutableMap.of("numHellos", "5"))).withTemplate(new URI("resource:///gobblin/runtime/instance/SimpleHelloWorldJob.template")).build();
    ResolvedJobSpec js1Resolved = new ResolvedJobSpec(js1);
    final String eventBusId = js1Resolved.getConfig().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    instanceLauncher.stopAsync();
    final int numHellos = js1Resolved.getConfig().getInt(HelloWorldSource.NUM_HELLOS_FULL_KEY);
    ArrayList<String> expectedEvents = new ArrayList<>();
    for (int i = 1; i <= numHellos; ++i) {
        expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
    }
    asserter.assertNextValuesEq(expectedEvents);
    asserter.close();
    instanceLauncher.awaitTerminated(5, TimeUnit.SECONDS);
}