Examples with ResolvedJobSpec org.apache.gobblin.runtime.job_spec.ResolvedJobSpec used on opensource projects

Example 1 with ResolvedJobSpec

use of org.apache.gobblin.runtime.job_spec.ResolvedJobSpec in project incubator-gobblin by apache.

the class EmbeddedGobblin method runAsync.

/**
 * Launch the Gobblin job asynchronously. This method will return when the Gobblin job has started.
 * @return a {@link JobExecutionDriver}. This object is a future that will resolve when the Gobblin job finishes.
 * @throws TimeoutException if the Gobblin job does not start within the launch timeout.
 */
@NotOnCli
public JobExecutionDriver runAsync() throws TimeoutException, InterruptedException {
    // Run function to distribute jars to workers in distributed mode
    this.distributeJarsFunction.run();
    Config sysProps = ConfigFactory.parseMap(this.builtConfigMap).withFallback(this.defaultSysConfig);
    Config userConfig = ConfigFactory.parseMap(this.userConfigMap);
    JobSpec jobSpec;
    if (this.jobFile.isPresent()) {
        try {
            Path jobFilePath = this.jobFile.get();
            PullFileLoader loader = new PullFileLoader(jobFilePath.getParent(), jobFilePath.getFileSystem(new Configuration()), PullFileLoader.DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS, PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS);
            Config jobConfig = userConfig.withFallback(loader.loadPullFile(jobFilePath, sysProps, false));
            ImmutableFSJobCatalog.JobSpecConverter converter = new ImmutableFSJobCatalog.JobSpecConverter(jobFilePath.getParent(), Optional.<String>absent());
            jobSpec = converter.apply(jobConfig);
        } catch (IOException ioe) {
            throw new RuntimeException("Failed to run embedded Gobblin.", ioe);
        }
    } else {
        Config finalConfig = userConfig.withFallback(sysProps);
        if (this.template != null) {
            try {
                finalConfig = this.template.getResolvedConfig(finalConfig);
            } catch (SpecNotFoundException | JobTemplate.TemplateException exc) {
                throw new RuntimeException(exc);
            }
        }
        jobSpec = this.specBuilder.withConfig(finalConfig).build();
    }
    ResolvedJobSpec resolvedJobSpec;
    try {
        resolvedJobSpec = new ResolvedJobSpec(jobSpec);
    } catch (SpecNotFoundException | JobTemplate.TemplateException exc) {
        throw new RuntimeException("Failed to resolved template.", exc);
    }
    final JobCatalog jobCatalog = new StaticJobCatalog(Optional.of(this.useLog), Lists.<JobSpec>newArrayList(resolvedJobSpec));
    SimpleGobblinInstanceEnvironment instanceEnvironment = new SimpleGobblinInstanceEnvironment("EmbeddedGobblinInstance", this.useLog, getSysConfig());
    StandardGobblinInstanceDriver.Builder builder = new StandardGobblinInstanceDriver.Builder(Optional.<GobblinInstanceEnvironment>of(instanceEnvironment)).withLog(this.useLog).withJobCatalog(jobCatalog).withImmediateJobScheduler();
    for (GobblinInstancePluginFactory plugin : this.plugins) {
        builder.addPlugin(plugin);
    }
    final GobblinInstanceDriver driver = builder.build();
    EmbeddedJobLifecycleListener listener = new EmbeddedJobLifecycleListener(this.useLog);
    driver.registerJobLifecycleListener(listener);
    driver.startAsync();
    boolean started = listener.awaitStarted(this.launchTimeout.getTimeout(), this.launchTimeout.getTimeUnit());
    if (!started) {
        log.warn("Timeout waiting for job to start. Aborting.");
        driver.stopAsync();
        driver.awaitTerminated(this.shutdownTimeout.getTimeout(), this.shutdownTimeout.getTimeUnit());
        throw new TimeoutException("Timeout waiting for job to start.");
    }
    final JobExecutionDriver jobDriver = listener.getJobDriver();
    // Stop the Gobblin instance driver when the job finishes.
    Futures.addCallback(jobDriver, new FutureCallback<JobExecutionResult>() {

        @Override
        public void onSuccess(@Nullable JobExecutionResult result) {
            stopGobblinInstanceDriver();
        }

        @Override
        public void onFailure(Throwable t) {
            stopGobblinInstanceDriver();
        }

        private void stopGobblinInstanceDriver() {
            try {
                driver.stopAsync();
                driver.awaitTerminated(EmbeddedGobblin.this.shutdownTimeout.getTimeout(), EmbeddedGobblin.this.shutdownTimeout.getTimeUnit());
            } catch (TimeoutException te) {
                log.error("Failed to shutdown Gobblin instance driver.");
            }
        }
    });
    return listener.getJobDriver();
}

Example 2 with ResolvedJobSpec

use of org.apache.gobblin.runtime.job_spec.ResolvedJobSpec in project incubator-gobblin by apache.

the class TestNonObservingFSJobCatalog method testCallbacks.

@Test
public void testCallbacks() throws Exception {
    this.jobConfigDir = java.nio.file.Files.createTempDirectory(String.format("gobblin-test_%s_job-conf", this.getClass().getSimpleName())).toFile();
    this.jobConfigDirPath = new Path(this.jobConfigDir.getPath());
    try (PrintWriter printWriter = new PrintWriter(new Path(jobConfigDirPath, "job3.template").toString(), "UTF-8")) {
        printWriter.println("param1 = value1");
        printWriter.println("param2 = value2");
    }
    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY, this.jobConfigDir.getPath());
    NonObservingFSJobCatalog cat = new NonObservingFSJobCatalog(ConfigUtils.propertiesToConfig(properties));
    cat.startAsync();
    cat.awaitRunning(10, TimeUnit.SECONDS);
    final Map<URI, JobSpec> specs = new Hashtable<>();
    JobCatalogListener l = Mockito.mock(JobCatalogListener.class);
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            JobSpec spec = (JobSpec) invocation.getArguments()[0];
            specs.put(spec.getUri(), spec);
            return null;
        }
    }).when(l).onAddJob(Mockito.any(JobSpec.class));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            JobSpec spec = (JobSpec) invocation.getArguments()[0];
            specs.put(spec.getUri(), spec);
            return null;
        }
    }).when(l).onUpdateJob(Mockito.any(JobSpec.class));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            URI uri = (URI) invocation.getArguments()[0];
            specs.remove(uri);
            return null;
        }
    }).when(l).onDeleteJob(Mockito.any(URI.class), Mockito.anyString());
    JobSpec js1_1 = JobSpec.builder("test_job1").withVersion("1").build();
    JobSpec js1_2 = JobSpec.builder("test_job1").withVersion("2").build();
    JobSpec js2 = JobSpec.builder("test_job2").withVersion("1").build();
    JobSpec js3 = JobSpec.builder("test_job3").withVersion("1").withTemplate(new URI("FS:///job3.template")).withConfig(ConfigBuilder.create().addPrimitive("job.template", "FS:///job3.template").build()).build();
    cat.addListener(l);
    cat.put(js1_1);
    Assert.assertTrue(specs.containsKey(js1_1.getUri()));
    JobSpec js1_1_notified = specs.get(js1_1.getUri());
    Assert.assertTrue(ConfigUtils.verifySubset(js1_1_notified.getConfig(), js1_1.getConfig()));
    Assert.assertEquals(js1_1.getVersion(), js1_1_notified.getVersion());
    cat.put(js1_2);
    Assert.assertTrue(specs.containsKey(js1_2.getUri()));
    JobSpec js1_2_notified = specs.get(js1_2.getUri());
    Assert.assertTrue(ConfigUtils.verifySubset(js1_2_notified.getConfig(), js1_2.getConfig()));
    Assert.assertEquals(js1_2.getVersion(), js1_2_notified.getVersion());
    cat.put(js2);
    Assert.assertTrue(specs.containsKey(js2.getUri()));
    JobSpec js2_notified = specs.get(js2.getUri());
    Assert.assertTrue(ConfigUtils.verifySubset(js2_notified.getConfig(), js2.getConfig()));
    Assert.assertEquals(js2.getVersion(), js2_notified.getVersion());
    cat.remove(js2.getUri());
    Assert.assertFalse(specs.containsKey(js2.getUri()));
    cat.put(js3);
    Assert.assertTrue(specs.containsKey(js3.getUri()));
    JobSpec js3_notified = specs.get(js3.getUri());
    Assert.assertTrue(ConfigUtils.verifySubset(js3_notified.getConfig(), js3.getConfig()));
    Assert.assertEquals(js3.getVersion(), js3_notified.getVersion());
    ResolvedJobSpec js3_resolved = new ResolvedJobSpec(js3_notified, cat);
    Assert.assertEquals(js3_resolved.getConfig().getString("param1"), "value1");
    Assert.assertEquals(js3_resolved.getConfig().getString("param2"), "value2");
    cat.stopAsync();
    cat.awaitTerminated(10, TimeUnit.SECONDS);
}

Example 3 with ResolvedJobSpec

use of org.apache.gobblin.runtime.job_spec.ResolvedJobSpec in project incubator-gobblin by apache.

the class TestFSJobCatalog method testCallbacks.

@Test
public void testCallbacks() throws Exception {
    this.jobConfigDir = java.nio.file.Files.createTempDirectory(String.format("gobblin-test_%s_job-conf", this.getClass().getSimpleName())).toFile();
    this.jobConfigDirPath = new Path(this.jobConfigDir.getPath());
    try (PrintWriter printWriter = new PrintWriter(new Path(jobConfigDirPath, "job3.template").toString(), "UTF-8")) {
        printWriter.println("param1 = value1");
        printWriter.println("param2 = value2");
    }
    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY, this.jobConfigDir.getPath());
    PathAlterationObserver observer = new PathAlterationObserver(this.jobConfigDirPath);
    /* Exposed the observer so that checkAndNotify can be manually invoked. */
    FSJobCatalog cat = new FSJobCatalog(ConfigUtils.propertiesToConfig(properties), observer);
    cat.startAsync();
    cat.awaitRunning(10, TimeUnit.SECONDS);
    final Map<URI, JobSpec> specs = new Hashtable<>();
    JobCatalogListener l = Mockito.mock(JobCatalogListener.class);
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            JobSpec spec = (JobSpec) invocation.getArguments()[0];
            specs.put(spec.getUri(), spec);
            return null;
        }
    }).when(l).onAddJob(Mockito.any(JobSpec.class));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            JobSpec spec = (JobSpec) invocation.getArguments()[0];
            specs.put(spec.getUri(), spec);
            return null;
        }
    }).when(l).onUpdateJob(Mockito.any(JobSpec.class));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            URI uri = (URI) invocation.getArguments()[0];
            specs.remove(uri);
            return null;
        }
    }).when(l).onDeleteJob(Mockito.any(URI.class), Mockito.anyString());
    JobSpec js1_1 = JobSpec.builder("test_job1").withVersion("1").build();
    JobSpec js1_2 = JobSpec.builder("test_job1").withVersion("2").build();
    JobSpec js2 = JobSpec.builder("test_job2").withVersion("1").build();
    JobSpec js3 = JobSpec.builder("test_job3").withVersion("1").withTemplate(new URI("FS:///job3.template")).withConfig(ConfigBuilder.create().addPrimitive("job.template", "FS:///job3.template").build()).build();
    cat.addListener(l);
    observer.initialize();
    cat.put(js1_1);
    // enough time for file creation.
    observer.checkAndNotify();
    Assert.assertTrue(specs.containsKey(js1_1.getUri()));
    JobSpec js1_1_notified = specs.get(js1_1.getUri());
    Assert.assertTrue(ConfigUtils.verifySubset(js1_1_notified.getConfig(), js1_1.getConfig()));
    Assert.assertEquals(js1_1.getVersion(), js1_1_notified.getVersion());
    // Linux system has too large granularity for the modification time.
    Thread.sleep(1000);
    cat.put(js1_2);
    // enough time for file replacement.
    observer.checkAndNotify();
    Assert.assertTrue(specs.containsKey(js1_2.getUri()));
    JobSpec js1_2_notified = specs.get(js1_2.getUri());
    Assert.assertTrue(ConfigUtils.verifySubset(js1_2_notified.getConfig(), js1_2.getConfig()));
    Assert.assertEquals(js1_2.getVersion(), js1_2_notified.getVersion());
    Thread.sleep(1000);
    cat.put(js2);
    observer.checkAndNotify();
    Assert.assertTrue(specs.containsKey(js2.getUri()));
    JobSpec js2_notified = specs.get(js2.getUri());
    Assert.assertTrue(ConfigUtils.verifySubset(js2_notified.getConfig(), js2.getConfig()));
    Assert.assertEquals(js2.getVersion(), js2_notified.getVersion());
    Thread.sleep(1000);
    cat.remove(js2.getUri());
    // enough time for file deletion.
    observer.checkAndNotify();
    Assert.assertFalse(specs.containsKey(js2.getUri()));
    Thread.sleep(1000);
    cat.put(js3);
    observer.checkAndNotify();
    Assert.assertTrue(specs.containsKey(js3.getUri()));
    JobSpec js3_notified = specs.get(js3.getUri());
    Assert.assertTrue(ConfigUtils.verifySubset(js3_notified.getConfig(), js3.getConfig()));
    Assert.assertEquals(js3.getVersion(), js3_notified.getVersion());
    ResolvedJobSpec js3_resolved = new ResolvedJobSpec(js3_notified, cat);
    Assert.assertEquals(js3_resolved.getConfig().getString("param1"), "value1");
    Assert.assertEquals(js3_resolved.getConfig().getString("param2"), "value2");
    cat.stopAsync();
    cat.awaitTerminated(10, TimeUnit.SECONDS);
}

Example 4 with ResolvedJobSpec

use of org.apache.gobblin.runtime.job_spec.ResolvedJobSpec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method buildJobSpec.

/**
 * Generate JobSpec based on the #templateURI that user specified.
 */
private JobSpec buildJobSpec(ServiceNode sourceNode, ServiceNode targetNode, URI templateURI, FlowSpec flowSpec) {
    JobSpec jobSpec;
    JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec, sourceNode, targetNode)).withConfig(flowSpec.getConfig()).withDescription(flowSpec.getDescription()).withVersion(flowSpec.getVersion());
    if (templateURI != null) {
        jobSpecBuilder.withTemplate(templateURI);
        try {
            jobSpec = new ResolvedJobSpec(jobSpecBuilder.build(), templateCatalog.get());
            log.info("Resolved JobSpec properties are: " + jobSpec.getConfigAsProperties());
        } catch (SpecNotFoundException | JobTemplate.TemplateException e) {
            throw new RuntimeException("Could not resolve template in JobSpec from TemplateCatalog", e);
        }
    } else {
        jobSpec = jobSpecBuilder.build();
        log.info("Unresolved JobSpec properties are: " + jobSpec.getConfigAsProperties());
    }
    // Remove schedule
    jobSpec.setConfig(jobSpec.getConfig().withoutPath(ConfigurationKeys.JOB_SCHEDULE_KEY));
    // Add job.name and job.group
    if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_NAME_KEY)) {
        jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.JOB_NAME_KEY, ConfigValueFactory.fromAnyRef(flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_NAME_KEY).unwrapped().toString() + "-" + sourceNode.getNodeName() + "-" + targetNode.getNodeName())));
    }
    if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_GROUP_KEY)) {
        jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.JOB_GROUP_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY)));
    }
    // Add flow execution id for this compilation
    long flowExecutionId = System.currentTimeMillis();
    jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, ConfigValueFactory.fromAnyRef(flowExecutionId)));
    // Reset properties in Spec from Config
    jobSpec.setConfigAsProperties(ConfigUtils.configToProperties(jobSpec.getConfig()));
    return jobSpec;
}

Example 5 with ResolvedJobSpec

use of org.apache.gobblin.runtime.job_spec.ResolvedJobSpec in project incubator-gobblin by apache.

the class BaseFlowToJobSpecCompiler method jobSpecGenerator.

/**
 * Naive implementation of generating jobSpec, which fetch the first available template,
 * in an exemplified single-hop FlowCompiler implementation.
 * @param flowSpec
 * @return
 */
protected JobSpec jobSpecGenerator(FlowSpec flowSpec) {
    JobSpec jobSpec;
    JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec)).withConfig(flowSpec.getConfig()).withDescription(flowSpec.getDescription()).withVersion(flowSpec.getVersion());
    if (flowSpec.getTemplateURIs().isPresent() && templateCatalog.isPresent()) {
        // Only first template uri will be honored for Identity
        jobSpecBuilder = jobSpecBuilder.withTemplate(flowSpec.getTemplateURIs().get().iterator().next());
        try {
            jobSpec = new ResolvedJobSpec(jobSpecBuilder.build(), templateCatalog.get());
            log.info("Resolved JobSpec properties are: " + jobSpec.getConfigAsProperties());
        } catch (SpecNotFoundException | JobTemplate.TemplateException e) {
            throw new RuntimeException("Could not resolve template in JobSpec from TemplateCatalog", e);
        }
    } else {
        jobSpec = jobSpecBuilder.build();
        log.info("Unresolved JobSpec properties are: " + jobSpec.getConfigAsProperties());
    }
    // Remove schedule
    jobSpec.setConfig(jobSpec.getConfig().withoutPath(ConfigurationKeys.JOB_SCHEDULE_KEY));
    // Add job.name and job.group
    if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_NAME_KEY)) {
        jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.JOB_NAME_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_NAME_KEY)));
    }
    if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_GROUP_KEY)) {
        jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.JOB_GROUP_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY)));
    }
    // Add flow execution id for this compilation
    long flowExecutionId = System.currentTimeMillis();
    jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, ConfigValueFactory.fromAnyRef(flowExecutionId)));
    // Reset properties in Spec from Config
    jobSpec.setConfigAsProperties(ConfigUtils.configToProperties(jobSpec.getConfig()));
    return jobSpec;
}