use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.
the class TestWorkUnitStreamSource method test.
/**
* This test uses a slow source to verify that we can stream work units through local job launcher, with available units
* being processes eagerly even if not all work units are available.
*/
@Test
public void test() throws Exception {
String eventBusId = UUID.randomUUID().toString();
MyListener listener = new MyListener();
EventBus eventBus = TestingEventBuses.getEventBus(eventBusId);
eventBus.register(listener);
EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("testStreamedSource").setConfiguration(EventBusPublishingTaskFactory.EVENTBUS_ID_KEY, eventBusId).setConfiguration(ConfigurationKeys.SOURCE_CLASS_KEY, MySource.class.getName()).setConfiguration(EventBusPublishingTaskFactory.Source.NUM_TASKS_KEY, "5");
JobExecutionDriver driver = embeddedGobblin.runAsync();
if (!listener.iteratorReady.tryAcquire(2, TimeUnit.SECONDS)) {
throw new RuntimeException("Failed to get start signal.");
}
Assert.assertFalse(listener.tasksRun.tryAcquire(50, TimeUnit.MILLISECONDS));
eventBus.post(new MySource.NextWorkUnit());
Assert.assertTrue(listener.tasksRun.tryAcquire(500, TimeUnit.MILLISECONDS));
Assert.assertFalse(listener.tasksRun.tryAcquire(50, TimeUnit.MILLISECONDS));
eventBus.post(new MySource.NextWorkUnit());
Assert.assertTrue(listener.tasksRun.tryAcquire(500, TimeUnit.MILLISECONDS));
Assert.assertFalse(listener.tasksRun.tryAcquire(50, TimeUnit.MILLISECONDS));
eventBus.post(new MySource.NextWorkUnit());
eventBus.post(new MySource.NextWorkUnit());
eventBus.post(new MySource.NextWorkUnit());
JobExecutionResult result = driver.get(5, TimeUnit.SECONDS);
Assert.assertTrue(result.isSuccessful());
SetMultimap<String, Integer> eventsSeen = listener.getEventsSeenMap();
Set<Integer> expected = Sets.newHashSet(0, 1, 2, 3, 4);
Assert.assertEquals(eventsSeen.get(EventBusPublishingTaskFactory.RUN_EVENT), expected);
Assert.assertEquals(eventsSeen.get(EventBusPublishingTaskFactory.COMMIT_EVENT), expected);
Assert.assertEquals(eventsSeen.get(EventBusPublishingTaskFactory.PUBLISH_EVENT), expected);
}
use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.
the class EmbeddedGobblinTest method testRunWithTemplate.
@Test
public void testRunWithTemplate() throws Exception {
String eventBusId = this.getClass().getName();
int numHellos = 5;
TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("TestJob").setTemplate(ResourceBasedJobTemplate.forResourcePath("templates/hello-world.template"));
embeddedGobblin.setConfiguration(ConfigurationKeys.WRITER_BUILDER_CLASS, GobblinTestEventBusWriter.Builder.class.getName());
embeddedGobblin.setConfiguration(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY, eventBusId);
embeddedGobblin.setConfiguration(HelloWorldSource.NUM_HELLOS_FULL_KEY, Integer.toString(numHellos));
JobExecutionResult result = embeddedGobblin.run();
Assert.assertTrue(result.isSuccessful());
ArrayList<String> expectedEvents = new ArrayList<>();
for (int i = 1; i <= numHellos; ++i) {
expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
}
asserter.assertNextValuesEq(expectedEvents);
asserter.close();
}
use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.
the class EmbeddedGobblinTest method testRunWithJobFile.
@Test
public void testRunWithJobFile() throws Exception {
String eventBusId = this.getClass().getName() + ".jobFileTest";
TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("TestJob").jobFile(getClass().getResource("/testJobs/helloWorld.conf").getPath());
embeddedGobblin.setConfiguration(ConfigurationKeys.WRITER_BUILDER_CLASS, GobblinTestEventBusWriter.Builder.class.getName());
embeddedGobblin.setConfiguration(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY, eventBusId);
JobExecutionResult result = embeddedGobblin.run();
Assert.assertTrue(result.isSuccessful());
ArrayList<String> expectedEvents = new ArrayList<>();
for (int i = 1; i <= 10; ++i) {
expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
}
asserter.assertNextValuesEq(expectedEvents);
asserter.close();
}
use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.
the class JobBrokerInjectionTest method testBrokerIsAcquiredAndShared.
@Test
public void testBrokerIsAcquiredAndShared() throws Exception {
StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitToJobCatalog");
instanceLauncherBuilder.driver();
StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
instanceLauncher.startAsync();
instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("brokerTest/SimpleHelloWorldJob.jobconf")).build();
final String eventBusId = js1.getConfig().resolve().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {
@Override
public void onJobLaunch(JobExecutionDriver jobDriver) {
super.onJobLaunch(jobDriver);
try {
jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
instance.getLog().error("Offer interrupted.");
}
}
});
instance.registerWeakJobLifecycleListener(js1Listener);
instance.getMutableJobCatalog().put(js1);
JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
Assert.assertNotNull(jobDriver);
JobExecutionResult jobResult = jobDriver.get(100000, TimeUnit.SECONDS);
Assert.assertTrue(jobResult.isSuccessful());
Queue<TestingEventBuses.Event> events = asserter.getEvents();
Set<Long> seenInstanceObjectIds = Sets.newHashSet();
Set<Long> seenJobObjectIds = Sets.newHashSet();
Set<Long> seenTaskObjectIds = Sets.newHashSet();
for (TestingEventBuses.Event event : events) {
MyRecord record = (MyRecord) event.getValue();
seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
seenJobObjectIds.add(record.getJobSharedObjectId());
seenTaskObjectIds.add(record.getTaskSharedObjectId());
}
// Should see same instance and job id (only 1 id in the set), but 5 different task ids for each task
Assert.assertEquals(seenInstanceObjectIds.size(), 1);
Assert.assertEquals(seenJobObjectIds.size(), 1);
Assert.assertEquals(seenTaskObjectIds.size(), 5);
asserter.clear();
instance.getMutableJobCatalog().remove(js1.getUri());
instance.getMutableJobCatalog().put(js1);
jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
Assert.assertNotNull(jobDriver);
jobResult = jobDriver.get(10, TimeUnit.SECONDS);
Assert.assertTrue(jobResult.isSuccessful());
events = asserter.getEvents();
for (TestingEventBuses.Event event : events) {
MyRecord record = (MyRecord) event.getValue();
seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
seenJobObjectIds.add(record.getJobSharedObjectId());
seenTaskObjectIds.add(record.getTaskSharedObjectId());
}
// A different job should produce a new shared object id
Assert.assertEquals(seenInstanceObjectIds.size(), 1);
Assert.assertEquals(seenJobObjectIds.size(), 2);
Assert.assertEquals(seenTaskObjectIds.size(), 10);
}
use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.
the class JobBrokerInjectionTest method launchJob.
private void launchJob(StandardGobblinInstanceLauncher instanceLauncher, JobSpec js1, GobblinInstanceDriver instance) throws TimeoutException, InterruptedException, ExecutionException {
JobExecutionDriver jobDriver = instance.getJobLauncher().launchJob(js1);
new Thread(jobDriver).run();
JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
Assert.assertTrue(jobResult.isSuccessful());
}
Aggregations