Examples with StandardGobblinInstanceDriver org.apache.gobblin.runtime.instance.StandardGobblinInstanceDriver used on opensource projects

Example 1 with StandardGobblinInstanceDriver

use of org.apache.gobblin.runtime.instance.StandardGobblinInstanceDriver in project incubator-gobblin by apache.

the class JobBrokerInjectionTest method testBrokerIsAcquiredAndShared.

@Test
public void testBrokerIsAcquiredAndShared() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitToJobCatalog");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("brokerTest/SimpleHelloWorldJob.jobconf")).build();
    final String eventBusId = js1.getConfig().resolve().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(100000, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    Queue<TestingEventBuses.Event> events = asserter.getEvents();
    Set<Long> seenInstanceObjectIds = Sets.newHashSet();
    Set<Long> seenJobObjectIds = Sets.newHashSet();
    Set<Long> seenTaskObjectIds = Sets.newHashSet();
    for (TestingEventBuses.Event event : events) {
        MyRecord record = (MyRecord) event.getValue();
        seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
        seenJobObjectIds.add(record.getJobSharedObjectId());
        seenTaskObjectIds.add(record.getTaskSharedObjectId());
    }
    // Should see same instance and job id (only 1 id in the set), but 5 different task ids for each task
    Assert.assertEquals(seenInstanceObjectIds.size(), 1);
    Assert.assertEquals(seenJobObjectIds.size(), 1);
    Assert.assertEquals(seenTaskObjectIds.size(), 5);
    asserter.clear();
    instance.getMutableJobCatalog().remove(js1.getUri());
    instance.getMutableJobCatalog().put(js1);
    jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    jobResult = jobDriver.get(10, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    events = asserter.getEvents();
    for (TestingEventBuses.Event event : events) {
        MyRecord record = (MyRecord) event.getValue();
        seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
        seenJobObjectIds.add(record.getJobSharedObjectId());
        seenTaskObjectIds.add(record.getTaskSharedObjectId());
    }
    // A different job should produce a new shared object id
    Assert.assertEquals(seenInstanceObjectIds.size(), 1);
    Assert.assertEquals(seenJobObjectIds.size(), 2);
    Assert.assertEquals(seenTaskObjectIds.size(), 10);
}