Search in sources :

Example 6 with JobExecutionResult

use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.

the class TestWorkUnitStreamSource method test.

/**
 * This test uses a slow source to verify that we can stream work units through local job launcher, with available units
 * being processes eagerly even if not all work units are available.
 */
@Test
public void test() throws Exception {
    String eventBusId = UUID.randomUUID().toString();
    MyListener listener = new MyListener();
    EventBus eventBus = TestingEventBuses.getEventBus(eventBusId);
    eventBus.register(listener);
    EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("testStreamedSource").setConfiguration(EventBusPublishingTaskFactory.EVENTBUS_ID_KEY, eventBusId).setConfiguration(ConfigurationKeys.SOURCE_CLASS_KEY, MySource.class.getName()).setConfiguration(EventBusPublishingTaskFactory.Source.NUM_TASKS_KEY, "5");
    JobExecutionDriver driver = embeddedGobblin.runAsync();
    if (!listener.iteratorReady.tryAcquire(2, TimeUnit.SECONDS)) {
        throw new RuntimeException("Failed to get start signal.");
    }
    Assert.assertFalse(listener.tasksRun.tryAcquire(50, TimeUnit.MILLISECONDS));
    eventBus.post(new MySource.NextWorkUnit());
    Assert.assertTrue(listener.tasksRun.tryAcquire(500, TimeUnit.MILLISECONDS));
    Assert.assertFalse(listener.tasksRun.tryAcquire(50, TimeUnit.MILLISECONDS));
    eventBus.post(new MySource.NextWorkUnit());
    Assert.assertTrue(listener.tasksRun.tryAcquire(500, TimeUnit.MILLISECONDS));
    Assert.assertFalse(listener.tasksRun.tryAcquire(50, TimeUnit.MILLISECONDS));
    eventBus.post(new MySource.NextWorkUnit());
    eventBus.post(new MySource.NextWorkUnit());
    eventBus.post(new MySource.NextWorkUnit());
    JobExecutionResult result = driver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(result.isSuccessful());
    SetMultimap<String, Integer> eventsSeen = listener.getEventsSeenMap();
    Set<Integer> expected = Sets.newHashSet(0, 1, 2, 3, 4);
    Assert.assertEquals(eventsSeen.get(EventBusPublishingTaskFactory.RUN_EVENT), expected);
    Assert.assertEquals(eventsSeen.get(EventBusPublishingTaskFactory.COMMIT_EVENT), expected);
    Assert.assertEquals(eventsSeen.get(EventBusPublishingTaskFactory.PUBLISH_EVENT), expected);
}
Also used : JobExecutionResult(org.apache.gobblin.runtime.api.JobExecutionResult) EventBus(com.google.common.eventbus.EventBus) EmbeddedGobblin(org.apache.gobblin.runtime.embedded.EmbeddedGobblin) JobExecutionDriver(org.apache.gobblin.runtime.api.JobExecutionDriver) Test(org.testng.annotations.Test)

Example 7 with JobExecutionResult

use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.

the class EmbeddedGobblinTest method testRunWithTemplate.

@Test
public void testRunWithTemplate() throws Exception {
    String eventBusId = this.getClass().getName();
    int numHellos = 5;
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("TestJob").setTemplate(ResourceBasedJobTemplate.forResourcePath("templates/hello-world.template"));
    embeddedGobblin.setConfiguration(ConfigurationKeys.WRITER_BUILDER_CLASS, GobblinTestEventBusWriter.Builder.class.getName());
    embeddedGobblin.setConfiguration(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY, eventBusId);
    embeddedGobblin.setConfiguration(HelloWorldSource.NUM_HELLOS_FULL_KEY, Integer.toString(numHellos));
    JobExecutionResult result = embeddedGobblin.run();
    Assert.assertTrue(result.isSuccessful());
    ArrayList<String> expectedEvents = new ArrayList<>();
    for (int i = 1; i <= numHellos; ++i) {
        expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
    }
    asserter.assertNextValuesEq(expectedEvents);
    asserter.close();
}
Also used : JobExecutionResult(org.apache.gobblin.runtime.api.JobExecutionResult) TestingEventBusAsserter(org.apache.gobblin.writer.test.TestingEventBusAsserter) ArrayList(java.util.ArrayList) Test(org.testng.annotations.Test)

Example 8 with JobExecutionResult

use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.

the class EmbeddedGobblinTest method testRunWithJobFile.

@Test
public void testRunWithJobFile() throws Exception {
    String eventBusId = this.getClass().getName() + ".jobFileTest";
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("TestJob").jobFile(getClass().getResource("/testJobs/helloWorld.conf").getPath());
    embeddedGobblin.setConfiguration(ConfigurationKeys.WRITER_BUILDER_CLASS, GobblinTestEventBusWriter.Builder.class.getName());
    embeddedGobblin.setConfiguration(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY, eventBusId);
    JobExecutionResult result = embeddedGobblin.run();
    Assert.assertTrue(result.isSuccessful());
    ArrayList<String> expectedEvents = new ArrayList<>();
    for (int i = 1; i <= 10; ++i) {
        expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
    }
    asserter.assertNextValuesEq(expectedEvents);
    asserter.close();
}
Also used : JobExecutionResult(org.apache.gobblin.runtime.api.JobExecutionResult) TestingEventBusAsserter(org.apache.gobblin.writer.test.TestingEventBusAsserter) ArrayList(java.util.ArrayList) Test(org.testng.annotations.Test)

Example 9 with JobExecutionResult

use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.

the class JobBrokerInjectionTest method testBrokerIsAcquiredAndShared.

@Test
public void testBrokerIsAcquiredAndShared() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitToJobCatalog");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("brokerTest/SimpleHelloWorldJob.jobconf")).build();
    final String eventBusId = js1.getConfig().resolve().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(100000, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    Queue<TestingEventBuses.Event> events = asserter.getEvents();
    Set<Long> seenInstanceObjectIds = Sets.newHashSet();
    Set<Long> seenJobObjectIds = Sets.newHashSet();
    Set<Long> seenTaskObjectIds = Sets.newHashSet();
    for (TestingEventBuses.Event event : events) {
        MyRecord record = (MyRecord) event.getValue();
        seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
        seenJobObjectIds.add(record.getJobSharedObjectId());
        seenTaskObjectIds.add(record.getTaskSharedObjectId());
    }
    // Should see same instance and job id (only 1 id in the set), but 5 different task ids for each task
    Assert.assertEquals(seenInstanceObjectIds.size(), 1);
    Assert.assertEquals(seenJobObjectIds.size(), 1);
    Assert.assertEquals(seenTaskObjectIds.size(), 5);
    asserter.clear();
    instance.getMutableJobCatalog().remove(js1.getUri());
    instance.getMutableJobCatalog().put(js1);
    jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    jobResult = jobDriver.get(10, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    events = asserter.getEvents();
    for (TestingEventBuses.Event event : events) {
        MyRecord record = (MyRecord) event.getValue();
        seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
        seenJobObjectIds.add(record.getJobSharedObjectId());
        seenTaskObjectIds.add(record.getTaskSharedObjectId());
    }
    // A different job should produce a new shared object id
    Assert.assertEquals(seenInstanceObjectIds.size(), 1);
    Assert.assertEquals(seenJobObjectIds.size(), 2);
    Assert.assertEquals(seenTaskObjectIds.size(), 10);
}
Also used : StandardGobblinInstanceDriver(org.apache.gobblin.runtime.instance.StandardGobblinInstanceDriver) DefaultJobLifecycleListenerImpl(org.apache.gobblin.runtime.std.DefaultJobLifecycleListenerImpl) JobExecutionResult(org.apache.gobblin.runtime.api.JobExecutionResult) TestingEventBuses(org.apache.gobblin.writer.test.TestingEventBuses) TestingEventBusAsserter(org.apache.gobblin.writer.test.TestingEventBusAsserter) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) StandardGobblinInstanceLauncher(org.apache.gobblin.runtime.instance.StandardGobblinInstanceLauncher) FilteredJobLifecycleListener(org.apache.gobblin.runtime.std.FilteredJobLifecycleListener) JobLifecycleListener(org.apache.gobblin.runtime.api.JobLifecycleListener) JobSpec(org.apache.gobblin.runtime.api.JobSpec) JobExecutionDriver(org.apache.gobblin.runtime.api.JobExecutionDriver) FilteredJobLifecycleListener(org.apache.gobblin.runtime.std.FilteredJobLifecycleListener) Test(org.testng.annotations.Test)

Example 10 with JobExecutionResult

use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.

the class JobBrokerInjectionTest method launchJob.

private void launchJob(StandardGobblinInstanceLauncher instanceLauncher, JobSpec js1, GobblinInstanceDriver instance) throws TimeoutException, InterruptedException, ExecutionException {
    JobExecutionDriver jobDriver = instance.getJobLauncher().launchJob(js1);
    new Thread(jobDriver).run();
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
}
Also used : JobExecutionResult(org.apache.gobblin.runtime.api.JobExecutionResult) JobExecutionDriver(org.apache.gobblin.runtime.api.JobExecutionDriver)

Aggregations

JobExecutionResult (org.apache.gobblin.runtime.api.JobExecutionResult)21 Test (org.testng.annotations.Test)18 EmbeddedGobblin (org.apache.gobblin.runtime.embedded.EmbeddedGobblin)12 File (java.io.File)9 JobExecutionDriver (org.apache.gobblin.runtime.api.JobExecutionDriver)8 GenericRecord (org.apache.avro.generic.GenericRecord)7 JobSpec (org.apache.gobblin.runtime.api.JobSpec)5 TestingEventBusAsserter (org.apache.gobblin.writer.test.TestingEventBusAsserter)5 ArrayList (java.util.ArrayList)4 ArrayBlockingQueue (java.util.concurrent.ArrayBlockingQueue)4 JobLifecycleListener (org.apache.gobblin.runtime.api.JobLifecycleListener)4 ResolvedJobSpec (org.apache.gobblin.runtime.job_spec.ResolvedJobSpec)4 DefaultJobLifecycleListenerImpl (org.apache.gobblin.runtime.std.DefaultJobLifecycleListenerImpl)4 FilteredJobLifecycleListener (org.apache.gobblin.runtime.std.FilteredJobLifecycleListener)4 EventBus (com.google.common.eventbus.EventBus)3 StandardGobblinInstanceDriver (org.apache.gobblin.runtime.instance.StandardGobblinInstanceDriver)2 Path (org.apache.hadoop.fs.Path)2 Config (com.typesafe.config.Config)1 IOException (java.io.IOException)1 URI (java.net.URI)1