Examples with JobSpec org.apache.gobblin.runtime.api.JobSpec used on opensource projects

Example 21 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class MockedKafkaJobMonitor method parseJobSpec.

@Override
public Collection<Either<JobSpec, URI>> parseJobSpec(byte[] message) throws IOException {
    try {
        String messageString = new String(message, Charsets.UTF_8);
        List<Either<JobSpec, URI>> jobSpecs = Lists.newArrayList();
        for (String oneInstruction : SPLITTER_COMMA.split(messageString)) {
            List<String> tokens = SPLITTER_COLON.splitToList(oneInstruction);
            if (tokens.get(0).equals(REMOVE)) {
                URI uri = new URI(tokens.get(1));
                jobSpecs.add(Either.<JobSpec, URI>right(uri));
            } else {
                URI uri = new URI(tokens.get(0));
                String version = tokens.get(1);
                JobSpec jobSpec = new JobSpec.Builder(uri).withConfig(ConfigFactory.empty()).withVersion(version).build();
                jobSpecs.add(Either.<JobSpec, URI>left(jobSpec));
            }
        }
        return jobSpecs;
    } catch (URISyntaxException use) {
        throw new IOException(use);
    }
}

Example 22 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class SLAEventKafkaJobMonitorTest method testParseJobSpec.

@Test
public void testParseJobSpec() throws Exception {
    SLAEventKafkaJobMonitor monitor = new SLAEventKafkaJobMonitor("topic", null, new URI("/base/URI"), HighLevelConsumerTest.getSimpleConfig(Optional.of(KafkaJobMonitor.KAFKA_JOB_MONITOR_PREFIX)), new NoopSchemaVersionWriter(), Optional.<Pattern>absent(), Optional.<Pattern>absent(), this.templateURI, ImmutableMap.of("metadataKey1", "key1"));
    monitor.buildMetricsContextAndMetrics();
    GobblinTrackingEvent event = createSLAEvent("DatasetPublish", new URI("/data/myDataset"), ImmutableMap.of("metadataKey1", "value1", "key1", "value2"));
    Collection<Either<JobSpec, URI>> jobSpecs = monitor.parseJobSpec(event);
    Assert.assertEquals(jobSpecs.size(), 1);
    JobSpec jobSpec = (JobSpec) jobSpecs.iterator().next().get();
    Assert.assertEquals(jobSpec.getUri(), new URI("/base/URI/data/myDataset"));
    Assert.assertEquals(jobSpec.getTemplateURI().get(), templateURI);
    // should insert configuration from metadata
    Assert.assertEquals(jobSpec.getConfig().getString("key1"), "value1");
    monitor.shutdownMetrics();
}

Example 23 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method testDirectToScheduler.

@Test
public /**
 * Test running of a job when submitted directly to the scheduler
 */
void testDirectToScheduler() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testDirectToScheduler");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("gobblin/runtime/instance/SimpleHelloWorldJob.jobconf")).build();
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    JobSpecRunnable js1Runnable = instance.createJobSpecRunnable(js1);
    instance.getJobScheduler().scheduleOnce(js1, js1Runnable);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    instanceLauncher.stopAsync();
    instanceLauncher.awaitTerminated(5, TimeUnit.SECONDS);
}

Example 24 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method testDirectToExecutionDriver.

@Test
public /**
 * Test running of a job when submitted directly to the execution driver
 */
void testDirectToExecutionDriver() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testDirectToExecutionDriver");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("gobblin/runtime/instance/SimpleHelloWorldJob.jobconf")).build();
    GobblinInstanceDriver instance = instanceLauncher.getDriver();
    final JobExecutionLauncher.StandardMetrics launcherMetrics = instance.getJobLauncher().getMetrics();
    AssertWithBackoff asb = new AssertWithBackoff().timeoutMs(100);
    checkLaunchJob(instanceLauncher, js1, instance);
    Assert.assertEquals(launcherMetrics.getNumJobsLaunched().getCount(), 1);
    Assert.assertEquals(launcherMetrics.getNumJobsCompleted().getCount(), 1);
    // Need to use assert with backoff because of race conditions with the callback that updates the
    // metrics
    asb.assertEquals(new Function<Void, Long>() {

        @Override
        public Long apply(Void input) {
            return launcherMetrics.getNumJobsCommitted().getCount();
        }
    }, 1l, "numJobsCommitted==1");
    Assert.assertEquals(launcherMetrics.getNumJobsFailed().getCount(), 0);
    Assert.assertEquals(launcherMetrics.getNumJobsRunning().getValue().intValue(), 0);
    checkLaunchJob(instanceLauncher, js1, instance);
    Assert.assertEquals(launcherMetrics.getNumJobsLaunched().getCount(), 2);
    Assert.assertEquals(launcherMetrics.getNumJobsCompleted().getCount(), 2);
    asb.assertEquals(new Function<Void, Long>() {

        @Override
        public Long apply(Void input) {
            return launcherMetrics.getNumJobsCommitted().getCount();
        }
    }, 2l, "numJobsCommitted==2");
    Assert.assertEquals(launcherMetrics.getNumJobsFailed().getCount(), 0);
    Assert.assertEquals(launcherMetrics.getNumJobsRunning().getValue().intValue(), 0);
}

Example 25 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class FSJobCatalogHelperTest method convertJobSpecList2PropList.

/**
 * Suppose in the testing routine, each JobSpec will at least have either config or properties.
 * @param jobConfigs
 * @return
 */
private List<Properties> convertJobSpecList2PropList(List<JobSpec> jobConfigs) {
    List<Properties> result = Lists.newArrayList();
    for (JobSpec js : jobConfigs) {
        Properties propToBeAdded;
        if (js.getConfigAsProperties() != null) {
            propToBeAdded = js.getConfigAsProperties();
        } else {
            propToBeAdded = ConfigUtils.configToProperties(js.getConfig());
        }
        // For the testing purpose, added it back when doing the comparison.
        propToBeAdded.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, js.getUri().toString());
        result.add(propToBeAdded);
    }
    return result;
}