use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.
the class MockedKafkaJobMonitor method parseJobSpec.
@Override
public Collection<Either<JobSpec, URI>> parseJobSpec(byte[] message) throws IOException {
try {
String messageString = new String(message, Charsets.UTF_8);
List<Either<JobSpec, URI>> jobSpecs = Lists.newArrayList();
for (String oneInstruction : SPLITTER_COMMA.split(messageString)) {
List<String> tokens = SPLITTER_COLON.splitToList(oneInstruction);
if (tokens.get(0).equals(REMOVE)) {
URI uri = new URI(tokens.get(1));
jobSpecs.add(Either.<JobSpec, URI>right(uri));
} else {
URI uri = new URI(tokens.get(0));
String version = tokens.get(1);
JobSpec jobSpec = new JobSpec.Builder(uri).withConfig(ConfigFactory.empty()).withVersion(version).build();
jobSpecs.add(Either.<JobSpec, URI>left(jobSpec));
}
}
return jobSpecs;
} catch (URISyntaxException use) {
throw new IOException(use);
}
}
use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.
the class SLAEventKafkaJobMonitorTest method testParseJobSpec.
@Test
public void testParseJobSpec() throws Exception {
SLAEventKafkaJobMonitor monitor = new SLAEventKafkaJobMonitor("topic", null, new URI("/base/URI"), HighLevelConsumerTest.getSimpleConfig(Optional.of(KafkaJobMonitor.KAFKA_JOB_MONITOR_PREFIX)), new NoopSchemaVersionWriter(), Optional.<Pattern>absent(), Optional.<Pattern>absent(), this.templateURI, ImmutableMap.of("metadataKey1", "key1"));
monitor.buildMetricsContextAndMetrics();
GobblinTrackingEvent event = createSLAEvent("DatasetPublish", new URI("/data/myDataset"), ImmutableMap.of("metadataKey1", "value1", "key1", "value2"));
Collection<Either<JobSpec, URI>> jobSpecs = monitor.parseJobSpec(event);
Assert.assertEquals(jobSpecs.size(), 1);
JobSpec jobSpec = (JobSpec) jobSpecs.iterator().next().get();
Assert.assertEquals(jobSpec.getUri(), new URI("/base/URI/data/myDataset"));
Assert.assertEquals(jobSpec.getTemplateURI().get(), templateURI);
// should insert configuration from metadata
Assert.assertEquals(jobSpec.getConfig().getString("key1"), "value1");
monitor.shutdownMetrics();
}
use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.
the class TestStandardGobblinInstanceLauncher method testDirectToScheduler.
@Test
public /**
* Test running of a job when submitted directly to the scheduler
*/
void testDirectToScheduler() throws Exception {
StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testDirectToScheduler");
instanceLauncherBuilder.driver();
StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
instanceLauncher.startAsync();
instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("gobblin/runtime/instance/SimpleHelloWorldJob.jobconf")).build();
final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {
@Override
public void onJobLaunch(JobExecutionDriver jobDriver) {
super.onJobLaunch(jobDriver);
try {
jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
instance.getLog().error("Offer interrupted.");
}
}
});
instance.registerWeakJobLifecycleListener(js1Listener);
JobSpecRunnable js1Runnable = instance.createJobSpecRunnable(js1);
instance.getJobScheduler().scheduleOnce(js1, js1Runnable);
JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
Assert.assertNotNull(jobDriver);
JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
Assert.assertTrue(jobResult.isSuccessful());
instanceLauncher.stopAsync();
instanceLauncher.awaitTerminated(5, TimeUnit.SECONDS);
}
use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.
the class TestStandardGobblinInstanceLauncher method testDirectToExecutionDriver.
@Test
public /**
* Test running of a job when submitted directly to the execution driver
*/
void testDirectToExecutionDriver() throws Exception {
StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testDirectToExecutionDriver");
instanceLauncherBuilder.driver();
StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
instanceLauncher.startAsync();
instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("gobblin/runtime/instance/SimpleHelloWorldJob.jobconf")).build();
GobblinInstanceDriver instance = instanceLauncher.getDriver();
final JobExecutionLauncher.StandardMetrics launcherMetrics = instance.getJobLauncher().getMetrics();
AssertWithBackoff asb = new AssertWithBackoff().timeoutMs(100);
checkLaunchJob(instanceLauncher, js1, instance);
Assert.assertEquals(launcherMetrics.getNumJobsLaunched().getCount(), 1);
Assert.assertEquals(launcherMetrics.getNumJobsCompleted().getCount(), 1);
// Need to use assert with backoff because of race conditions with the callback that updates the
// metrics
asb.assertEquals(new Function<Void, Long>() {
@Override
public Long apply(Void input) {
return launcherMetrics.getNumJobsCommitted().getCount();
}
}, 1l, "numJobsCommitted==1");
Assert.assertEquals(launcherMetrics.getNumJobsFailed().getCount(), 0);
Assert.assertEquals(launcherMetrics.getNumJobsRunning().getValue().intValue(), 0);
checkLaunchJob(instanceLauncher, js1, instance);
Assert.assertEquals(launcherMetrics.getNumJobsLaunched().getCount(), 2);
Assert.assertEquals(launcherMetrics.getNumJobsCompleted().getCount(), 2);
asb.assertEquals(new Function<Void, Long>() {
@Override
public Long apply(Void input) {
return launcherMetrics.getNumJobsCommitted().getCount();
}
}, 2l, "numJobsCommitted==2");
Assert.assertEquals(launcherMetrics.getNumJobsFailed().getCount(), 0);
Assert.assertEquals(launcherMetrics.getNumJobsRunning().getValue().intValue(), 0);
}
use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.
the class FSJobCatalogHelperTest method convertJobSpecList2PropList.
/**
* Suppose in the testing routine, each JobSpec will at least have either config or properties.
* @param jobConfigs
* @return
*/
private List<Properties> convertJobSpecList2PropList(List<JobSpec> jobConfigs) {
List<Properties> result = Lists.newArrayList();
for (JobSpec js : jobConfigs) {
Properties propToBeAdded;
if (js.getConfigAsProperties() != null) {
propToBeAdded = js.getConfigAsProperties();
} else {
propToBeAdded = ConfigUtils.configToProperties(js.getConfig());
}
// For the testing purpose, added it back when doing the comparison.
propToBeAdded.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, js.getUri().toString());
result.add(propToBeAdded);
}
return result;
}
Aggregations