Search in sources :

Example 1 with GobblinInstanceEnvironment

use of org.apache.gobblin.runtime.api.GobblinInstanceEnvironment in project incubator-gobblin by apache.

the class EmbeddedGobblin method runAsync.

/**
 * Launch the Gobblin job asynchronously. This method will return when the Gobblin job has started.
 * @return a {@link JobExecutionDriver}. This object is a future that will resolve when the Gobblin job finishes.
 * @throws TimeoutException if the Gobblin job does not start within the launch timeout.
 */
@NotOnCli
public JobExecutionDriver runAsync() throws TimeoutException, InterruptedException {
    // Run function to distribute jars to workers in distributed mode
    this.distributeJarsFunction.run();
    Config sysProps = ConfigFactory.parseMap(this.builtConfigMap).withFallback(this.defaultSysConfig);
    Config userConfig = ConfigFactory.parseMap(this.userConfigMap);
    JobSpec jobSpec;
    if (this.jobFile.isPresent()) {
        try {
            Path jobFilePath = this.jobFile.get();
            PullFileLoader loader = new PullFileLoader(jobFilePath.getParent(), jobFilePath.getFileSystem(new Configuration()), PullFileLoader.DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS, PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS);
            Config jobConfig = userConfig.withFallback(loader.loadPullFile(jobFilePath, sysProps, false));
            ImmutableFSJobCatalog.JobSpecConverter converter = new ImmutableFSJobCatalog.JobSpecConverter(jobFilePath.getParent(), Optional.<String>absent());
            jobSpec = converter.apply(jobConfig);
        } catch (IOException ioe) {
            throw new RuntimeException("Failed to run embedded Gobblin.", ioe);
        }
    } else {
        Config finalConfig = userConfig.withFallback(sysProps);
        if (this.template != null) {
            try {
                finalConfig = this.template.getResolvedConfig(finalConfig);
            } catch (SpecNotFoundException | JobTemplate.TemplateException exc) {
                throw new RuntimeException(exc);
            }
        }
        jobSpec = this.specBuilder.withConfig(finalConfig).build();
    }
    ResolvedJobSpec resolvedJobSpec;
    try {
        resolvedJobSpec = new ResolvedJobSpec(jobSpec);
    } catch (SpecNotFoundException | JobTemplate.TemplateException exc) {
        throw new RuntimeException("Failed to resolved template.", exc);
    }
    final JobCatalog jobCatalog = new StaticJobCatalog(Optional.of(this.useLog), Lists.<JobSpec>newArrayList(resolvedJobSpec));
    SimpleGobblinInstanceEnvironment instanceEnvironment = new SimpleGobblinInstanceEnvironment("EmbeddedGobblinInstance", this.useLog, getSysConfig());
    StandardGobblinInstanceDriver.Builder builder = new StandardGobblinInstanceDriver.Builder(Optional.<GobblinInstanceEnvironment>of(instanceEnvironment)).withLog(this.useLog).withJobCatalog(jobCatalog).withImmediateJobScheduler();
    for (GobblinInstancePluginFactory plugin : this.plugins) {
        builder.addPlugin(plugin);
    }
    final GobblinInstanceDriver driver = builder.build();
    EmbeddedJobLifecycleListener listener = new EmbeddedJobLifecycleListener(this.useLog);
    driver.registerJobLifecycleListener(listener);
    driver.startAsync();
    boolean started = listener.awaitStarted(this.launchTimeout.getTimeout(), this.launchTimeout.getTimeUnit());
    if (!started) {
        log.warn("Timeout waiting for job to start. Aborting.");
        driver.stopAsync();
        driver.awaitTerminated(this.shutdownTimeout.getTimeout(), this.shutdownTimeout.getTimeUnit());
        throw new TimeoutException("Timeout waiting for job to start.");
    }
    final JobExecutionDriver jobDriver = listener.getJobDriver();
    // Stop the Gobblin instance driver when the job finishes.
    Futures.addCallback(jobDriver, new FutureCallback<JobExecutionResult>() {

        @Override
        public void onSuccess(@Nullable JobExecutionResult result) {
            stopGobblinInstanceDriver();
        }

        @Override
        public void onFailure(Throwable t) {
            stopGobblinInstanceDriver();
        }

        private void stopGobblinInstanceDriver() {
            try {
                driver.stopAsync();
                driver.awaitTerminated(EmbeddedGobblin.this.shutdownTimeout.getTimeout(), EmbeddedGobblin.this.shutdownTimeout.getTimeUnit());
            } catch (TimeoutException te) {
                log.error("Failed to shutdown Gobblin instance driver.");
            }
        }
    });
    return listener.getJobDriver();
}
Also used : ImmutableFSJobCatalog(org.apache.gobblin.runtime.job_catalog.ImmutableFSJobCatalog) Configuration(org.apache.hadoop.conf.Configuration) Config(com.typesafe.config.Config) StandardGobblinInstanceDriver(org.apache.gobblin.runtime.instance.StandardGobblinInstanceDriver) GobblinInstanceDriver(org.apache.gobblin.runtime.api.GobblinInstanceDriver) StandardGobblinInstanceDriver(org.apache.gobblin.runtime.instance.StandardGobblinInstanceDriver) StaticJobCatalog(org.apache.gobblin.runtime.job_catalog.StaticJobCatalog) JobCatalog(org.apache.gobblin.runtime.api.JobCatalog) ImmutableFSJobCatalog(org.apache.gobblin.runtime.job_catalog.ImmutableFSJobCatalog) GobblinInstanceEnvironment(org.apache.gobblin.runtime.api.GobblinInstanceEnvironment) SimpleGobblinInstanceEnvironment(org.apache.gobblin.runtime.instance.SimpleGobblinInstanceEnvironment) SpecNotFoundException(org.apache.gobblin.runtime.api.SpecNotFoundException) StaticJobCatalog(org.apache.gobblin.runtime.job_catalog.StaticJobCatalog) ResolvedJobSpec(org.apache.gobblin.runtime.job_spec.ResolvedJobSpec) GobblinInstancePluginFactory(org.apache.gobblin.runtime.api.GobblinInstancePluginFactory) JobExecutionDriver(org.apache.gobblin.runtime.api.JobExecutionDriver) TimeoutException(java.util.concurrent.TimeoutException) Path(org.apache.hadoop.fs.Path) PullFileLoader(org.apache.gobblin.util.PullFileLoader) IOException(java.io.IOException) SimpleGobblinInstanceEnvironment(org.apache.gobblin.runtime.instance.SimpleGobblinInstanceEnvironment) JobExecutionResult(org.apache.gobblin.runtime.api.JobExecutionResult) ResolvedJobSpec(org.apache.gobblin.runtime.job_spec.ResolvedJobSpec) JobSpec(org.apache.gobblin.runtime.api.JobSpec) NotOnCli(org.apache.gobblin.runtime.cli.NotOnCli)

Example 2 with GobblinInstanceEnvironment

use of org.apache.gobblin.runtime.api.GobblinInstanceEnvironment in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceDriver method testBuilder.

@Test
public void testBuilder() {
    Config instanceCfg = ConfigFactory.parseMap(ImmutableMap.<String, Object>builder().put(StandardGobblinInstanceDriver.PLUGINS_FULL_KEY, "fake1").put(EmailNotificationPlugin.EMAIL_NOTIFICATIONS_DISABLED_KEY, Boolean.valueOf(true)).build());
    GobblinInstanceEnvironment mockEnv = Mockito.mock(GobblinInstanceEnvironment.class);
    Mockito.when(mockEnv.getSysConfig()).thenReturn(DefaultConfigurableImpl.createFromConfig(instanceCfg));
    StandardGobblinInstanceDriver.Builder builder = StandardGobblinInstanceDriver.builder().withInstanceEnvironment(mockEnv).addPlugin(new FakePluginFactory2());
    List<GobblinInstancePluginFactory> plugins = builder.getPlugins();
    Assert.assertEquals(plugins.size(), 2);
    Set<Class<?>> pluginClasses = new HashSet<>();
    pluginClasses.add(plugins.get(0).getClass());
    pluginClasses.add(plugins.get(1).getClass());
    Assert.assertTrue(pluginClasses.contains(FakePluginFactory1.class));
    Assert.assertTrue(pluginClasses.contains(FakePluginFactory2.class));
}
Also used : Config(com.typesafe.config.Config) GobblinInstanceEnvironment(org.apache.gobblin.runtime.api.GobblinInstanceEnvironment) GobblinInstancePluginFactory(org.apache.gobblin.runtime.api.GobblinInstancePluginFactory) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Aggregations

Config (com.typesafe.config.Config)2 GobblinInstanceEnvironment (org.apache.gobblin.runtime.api.GobblinInstanceEnvironment)2 GobblinInstancePluginFactory (org.apache.gobblin.runtime.api.GobblinInstancePluginFactory)2 IOException (java.io.IOException)1 HashSet (java.util.HashSet)1 TimeoutException (java.util.concurrent.TimeoutException)1 GobblinInstanceDriver (org.apache.gobblin.runtime.api.GobblinInstanceDriver)1 JobCatalog (org.apache.gobblin.runtime.api.JobCatalog)1 JobExecutionDriver (org.apache.gobblin.runtime.api.JobExecutionDriver)1 JobExecutionResult (org.apache.gobblin.runtime.api.JobExecutionResult)1 JobSpec (org.apache.gobblin.runtime.api.JobSpec)1 SpecNotFoundException (org.apache.gobblin.runtime.api.SpecNotFoundException)1 NotOnCli (org.apache.gobblin.runtime.cli.NotOnCli)1 SimpleGobblinInstanceEnvironment (org.apache.gobblin.runtime.instance.SimpleGobblinInstanceEnvironment)1 StandardGobblinInstanceDriver (org.apache.gobblin.runtime.instance.StandardGobblinInstanceDriver)1 ImmutableFSJobCatalog (org.apache.gobblin.runtime.job_catalog.ImmutableFSJobCatalog)1 StaticJobCatalog (org.apache.gobblin.runtime.job_catalog.StaticJobCatalog)1 ResolvedJobSpec (org.apache.gobblin.runtime.job_spec.ResolvedJobSpec)1 PullFileLoader (org.apache.gobblin.util.PullFileLoader)1 Configuration (org.apache.hadoop.conf.Configuration)1