use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class StreamingJobConfigurationManager method fetchJobSpecs.
private void fetchJobSpecs() throws ExecutionException, InterruptedException {
List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutor.Verb, Spec>>) this.specConsumer.changedSpecs().get();
// propagate thread interruption so that caller will exit from loop
if (Thread.interrupted()) {
throw new InterruptedException();
}
for (Pair<SpecExecutor.Verb, Spec> entry : changesSpecs) {
SpecExecutor.Verb verb = entry.getKey();
if (verb.equals(SpecExecutor.Verb.ADD)) {
// Handle addition
JobSpec jobSpec = (JobSpec) entry.getValue();
postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties());
} else if (verb.equals(SpecExecutor.Verb.UPDATE)) {
// Handle update
JobSpec jobSpec = (JobSpec) entry.getValue();
postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties());
} else if (verb.equals(SpecExecutor.Verb.DELETE)) {
// Handle delete
Spec anonymousSpec = (Spec) entry.getValue();
postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties());
}
}
}
use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class StreamingKafkaSpecExecutorTest method testAddSpec.
@Test
public void testAddSpec() throws Exception {
_closer = Closer.create();
_properties = new Properties();
// Properties for Producer
_properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, TOPIC);
_properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", _kafkaBrokers);
_properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
// Properties for Consumer
_properties.setProperty("jobSpecMonitor.kafka.zookeeper.connect", zkConnect);
_properties.setProperty(SimpleKafkaSpecExecutor.SPEC_KAFKA_TOPICS_KEY, TOPIC);
_properties.setProperty("gobblin.cluster.jobconf.fullyQualifiedPath", _JOBS_DIR_PATH);
Config config = ConfigUtils.propertiesToConfig(_properties);
// SEI Producer
_seip = _closer.register(new SimpleKafkaSpecProducer(config));
String addedSpecUriString = "/foo/bar/addedSpec";
Spec spec = initJobSpec(addedSpecUriString);
WriteResponse writeResponse = (WriteResponse) _seip.addSpec(spec).get();
log.info("WriteResponse: " + writeResponse);
_jobCatalog = new NonObservingFSJobCatalog(config.getConfig("gobblin.cluster"));
_jobCatalog.startAsync().awaitRunning();
// SEI Consumer
_seic = _closer.register(new StreamingKafkaSpecConsumer(config, _jobCatalog));
_seic.startAsync().awaitRunning();
List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.ADD), "Verb did not match");
Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(addedSpecUriString), "Expected URI did not match");
Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}
use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class StreamingKafkaSpecExecutorTest method testDeleteSpec.
@Test(dependsOnMethods = "testUpdateSpec")
public void testDeleteSpec() throws Exception {
// delete needs to be on a job spec that exists to get notification
String deletedSpecUriString = "/foo/bar/addedSpec";
WriteResponse writeResponse = (WriteResponse) _seip.deleteSpec(new URI(deletedSpecUriString)).get();
log.info("WriteResponse: " + writeResponse);
List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match");
Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(deletedSpecUriString), "Expected URI did not match");
Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}
use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class SimpleKafkaSpecExecutorTest method testDeleteSpec.
@Test(dependsOnMethods = "testUpdateSpec")
public void testDeleteSpec() throws Exception {
String deletedSpecUriString = "/foo/bar/deletedSpec";
WriteResponse writeResponse = (WriteResponse) _seip.deleteSpec(new URI(deletedSpecUriString)).get();
log.info("WriteResponse: " + writeResponse);
try {
Thread.sleep(1000);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match");
Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(deletedSpecUriString), "Expected URI did not match");
Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}
use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class SimpleKafkaSpecExecutorTest method testUpdateSpec.
@Test(dependsOnMethods = "testAddSpec")
public void testUpdateSpec() throws Exception {
String updatedSpecUriString = "/foo/bar/updatedSpec";
Spec spec = initJobSpec(updatedSpecUriString);
WriteResponse writeResponse = (WriteResponse) _seip.updateSpec(spec).get();
log.info("WriteResponse: " + writeResponse);
try {
Thread.sleep(1000);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.UPDATE), "Verb did not match");
Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(updatedSpecUriString), "Expected URI did not match");
Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}
Aggregations