Examples with JobExecutionDriver org.apache.gobblin.runtime.api.JobExecutionDriver used on opensource projects

Example 1 with JobExecutionDriver

use of org.apache.gobblin.runtime.api.JobExecutionDriver in project incubator-gobblin by apache.

the class EmbeddedGobblin method runAsync.

/**
 * Launch the Gobblin job asynchronously. This method will return when the Gobblin job has started.
 * @return a {@link JobExecutionDriver}. This object is a future that will resolve when the Gobblin job finishes.
 * @throws TimeoutException if the Gobblin job does not start within the launch timeout.
 */
@NotOnCli
public JobExecutionDriver runAsync() throws TimeoutException, InterruptedException {
    // Run function to distribute jars to workers in distributed mode
    this.distributeJarsFunction.run();
    Config sysProps = ConfigFactory.parseMap(this.builtConfigMap).withFallback(this.defaultSysConfig);
    Config userConfig = ConfigFactory.parseMap(this.userConfigMap);
    JobSpec jobSpec;
    if (this.jobFile.isPresent()) {
        try {
            Path jobFilePath = this.jobFile.get();
            PullFileLoader loader = new PullFileLoader(jobFilePath.getParent(), jobFilePath.getFileSystem(new Configuration()), PullFileLoader.DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS, PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS);
            Config jobConfig = userConfig.withFallback(loader.loadPullFile(jobFilePath, sysProps, false));
            ImmutableFSJobCatalog.JobSpecConverter converter = new ImmutableFSJobCatalog.JobSpecConverter(jobFilePath.getParent(), Optional.<String>absent());
            jobSpec = converter.apply(jobConfig);
        } catch (IOException ioe) {
            throw new RuntimeException("Failed to run embedded Gobblin.", ioe);
        }
    } else {
        Config finalConfig = userConfig.withFallback(sysProps);
        if (this.template != null) {
            try {
                finalConfig = this.template.getResolvedConfig(finalConfig);
            } catch (SpecNotFoundException | JobTemplate.TemplateException exc) {
                throw new RuntimeException(exc);
            }
        }
        jobSpec = this.specBuilder.withConfig(finalConfig).build();
    }
    ResolvedJobSpec resolvedJobSpec;
    try {
        resolvedJobSpec = new ResolvedJobSpec(jobSpec);
    } catch (SpecNotFoundException | JobTemplate.TemplateException exc) {
        throw new RuntimeException("Failed to resolved template.", exc);
    }
    final JobCatalog jobCatalog = new StaticJobCatalog(Optional.of(this.useLog), Lists.<JobSpec>newArrayList(resolvedJobSpec));
    SimpleGobblinInstanceEnvironment instanceEnvironment = new SimpleGobblinInstanceEnvironment("EmbeddedGobblinInstance", this.useLog, getSysConfig());
    StandardGobblinInstanceDriver.Builder builder = new StandardGobblinInstanceDriver.Builder(Optional.<GobblinInstanceEnvironment>of(instanceEnvironment)).withLog(this.useLog).withJobCatalog(jobCatalog).withImmediateJobScheduler();
    for (GobblinInstancePluginFactory plugin : this.plugins) {
        builder.addPlugin(plugin);
    }
    final GobblinInstanceDriver driver = builder.build();
    EmbeddedJobLifecycleListener listener = new EmbeddedJobLifecycleListener(this.useLog);
    driver.registerJobLifecycleListener(listener);
    driver.startAsync();
    boolean started = listener.awaitStarted(this.launchTimeout.getTimeout(), this.launchTimeout.getTimeUnit());
    if (!started) {
        log.warn("Timeout waiting for job to start. Aborting.");
        driver.stopAsync();
        driver.awaitTerminated(this.shutdownTimeout.getTimeout(), this.shutdownTimeout.getTimeUnit());
        throw new TimeoutException("Timeout waiting for job to start.");
    }
    final JobExecutionDriver jobDriver = listener.getJobDriver();
    // Stop the Gobblin instance driver when the job finishes.
    Futures.addCallback(jobDriver, new FutureCallback<JobExecutionResult>() {

        @Override
        public void onSuccess(@Nullable JobExecutionResult result) {
            stopGobblinInstanceDriver();
        }

        @Override
        public void onFailure(Throwable t) {
            stopGobblinInstanceDriver();
        }

        private void stopGobblinInstanceDriver() {
            try {
                driver.stopAsync();
                driver.awaitTerminated(EmbeddedGobblin.this.shutdownTimeout.getTimeout(), EmbeddedGobblin.this.shutdownTimeout.getTimeUnit());
            } catch (TimeoutException te) {
                log.error("Failed to shutdown Gobblin instance driver.");
            }
        }
    });
    return listener.getJobDriver();
}

Example 2 with JobExecutionDriver

use of org.apache.gobblin.runtime.api.JobExecutionDriver in project incubator-gobblin by apache.

the class TestWorkUnitStreamSource method test.

/**
 * This test uses a slow source to verify that we can stream work units through local job launcher, with available units
 * being processes eagerly even if not all work units are available.
 */
@Test
public void test() throws Exception {
    String eventBusId = UUID.randomUUID().toString();
    MyListener listener = new MyListener();
    EventBus eventBus = TestingEventBuses.getEventBus(eventBusId);
    eventBus.register(listener);
    EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("testStreamedSource").setConfiguration(EventBusPublishingTaskFactory.EVENTBUS_ID_KEY, eventBusId).setConfiguration(ConfigurationKeys.SOURCE_CLASS_KEY, MySource.class.getName()).setConfiguration(EventBusPublishingTaskFactory.Source.NUM_TASKS_KEY, "5");
    JobExecutionDriver driver = embeddedGobblin.runAsync();
    if (!listener.iteratorReady.tryAcquire(2, TimeUnit.SECONDS)) {
        throw new RuntimeException("Failed to get start signal.");
    }
    Assert.assertFalse(listener.tasksRun.tryAcquire(50, TimeUnit.MILLISECONDS));
    eventBus.post(new MySource.NextWorkUnit());
    Assert.assertTrue(listener.tasksRun.tryAcquire(500, TimeUnit.MILLISECONDS));
    Assert.assertFalse(listener.tasksRun.tryAcquire(50, TimeUnit.MILLISECONDS));
    eventBus.post(new MySource.NextWorkUnit());
    Assert.assertTrue(listener.tasksRun.tryAcquire(500, TimeUnit.MILLISECONDS));
    Assert.assertFalse(listener.tasksRun.tryAcquire(50, TimeUnit.MILLISECONDS));
    eventBus.post(new MySource.NextWorkUnit());
    eventBus.post(new MySource.NextWorkUnit());
    eventBus.post(new MySource.NextWorkUnit());
    JobExecutionResult result = driver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(result.isSuccessful());
    SetMultimap<String, Integer> eventsSeen = listener.getEventsSeenMap();
    Set<Integer> expected = Sets.newHashSet(0, 1, 2, 3, 4);
    Assert.assertEquals(eventsSeen.get(EventBusPublishingTaskFactory.RUN_EVENT), expected);
    Assert.assertEquals(eventsSeen.get(EventBusPublishingTaskFactory.COMMIT_EVENT), expected);
    Assert.assertEquals(eventsSeen.get(EventBusPublishingTaskFactory.PUBLISH_EVENT), expected);
}

Example 3 with JobExecutionDriver

use of org.apache.gobblin.runtime.api.JobExecutionDriver in project incubator-gobblin by apache.

the class JobBrokerInjectionTest method testBrokerIsAcquiredAndShared.

@Test
public void testBrokerIsAcquiredAndShared() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitToJobCatalog");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("brokerTest/SimpleHelloWorldJob.jobconf")).build();
    final String eventBusId = js1.getConfig().resolve().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(100000, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    Queue<TestingEventBuses.Event> events = asserter.getEvents();
    Set<Long> seenInstanceObjectIds = Sets.newHashSet();
    Set<Long> seenJobObjectIds = Sets.newHashSet();
    Set<Long> seenTaskObjectIds = Sets.newHashSet();
    for (TestingEventBuses.Event event : events) {
        MyRecord record = (MyRecord) event.getValue();
        seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
        seenJobObjectIds.add(record.getJobSharedObjectId());
        seenTaskObjectIds.add(record.getTaskSharedObjectId());
    }
    // Should see same instance and job id (only 1 id in the set), but 5 different task ids for each task
    Assert.assertEquals(seenInstanceObjectIds.size(), 1);
    Assert.assertEquals(seenJobObjectIds.size(), 1);
    Assert.assertEquals(seenTaskObjectIds.size(), 5);
    asserter.clear();
    instance.getMutableJobCatalog().remove(js1.getUri());
    instance.getMutableJobCatalog().put(js1);
    jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    jobResult = jobDriver.get(10, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    events = asserter.getEvents();
    for (TestingEventBuses.Event event : events) {
        MyRecord record = (MyRecord) event.getValue();
        seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
        seenJobObjectIds.add(record.getJobSharedObjectId());
        seenTaskObjectIds.add(record.getTaskSharedObjectId());
    }
    // A different job should produce a new shared object id
    Assert.assertEquals(seenInstanceObjectIds.size(), 1);
    Assert.assertEquals(seenJobObjectIds.size(), 2);
    Assert.assertEquals(seenTaskObjectIds.size(), 10);
}

Example 4 with JobExecutionDriver

use of org.apache.gobblin.runtime.api.JobExecutionDriver in project incubator-gobblin by apache.

the class JobBrokerInjectionTest method launchJob.

private void launchJob(StandardGobblinInstanceLauncher instanceLauncher, JobSpec js1, GobblinInstanceDriver instance) throws TimeoutException, InterruptedException, ExecutionException {
    JobExecutionDriver jobDriver = instance.getJobLauncher().launchJob(js1);
    new Thread(jobDriver).run();
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
}

Example 5 with JobExecutionDriver

use of org.apache.gobblin.runtime.api.JobExecutionDriver in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method checkLaunchJob.

private void checkLaunchJob(StandardGobblinInstanceLauncher instanceLauncher, JobSpec js1, GobblinInstanceDriver instance) throws TimeoutException, InterruptedException, ExecutionException {
    JobExecutionDriver jobDriver = instance.getJobLauncher().launchJob(js1);
    new Thread(jobDriver).run();
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    instanceLauncher.stopAsync();
    instanceLauncher.awaitTerminated(5, TimeUnit.SECONDS);
    Assert.assertEquals(instance.getMetrics().getUpFlag().getValue().intValue(), 0);
    Assert.assertEquals(instance.getMetrics().getUptimeMs().getValue().longValue(), 0);
}