use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class ScheduledJobConfigurationManager method fetchJobSpecs.
/**
* TODO: Change cluster code to handle Spec. Right now all job properties are needed to be in config and template is not honored
* TODO: Materialized JobSpec and make use of ResolvedJobSpec
* @throws ExecutionException
* @throws InterruptedException
*/
private void fetchJobSpecs() throws ExecutionException, InterruptedException {
List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutor.Verb, Spec>>) this._specConsumer.changedSpecs().get();
for (Pair<SpecExecutor.Verb, Spec> entry : changesSpecs) {
SpecExecutor.Verb verb = entry.getKey();
if (verb.equals(SpecExecutor.Verb.ADD)) {
// Handle addition
JobSpec jobSpec = (JobSpec) entry.getValue();
postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties());
jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue());
} else if (verb.equals(SpecExecutor.Verb.UPDATE)) {
// Handle update
JobSpec jobSpec = (JobSpec) entry.getValue();
postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties());
jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue());
} else if (verb.equals(SpecExecutor.Verb.DELETE)) {
// Handle delete
Spec anonymousSpec = (Spec) entry.getValue();
postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties());
jobSpecs.remove(entry.getValue().getUri());
}
}
}
use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class StreamingKafkaSpecExecutorTest method testUpdateSpec.
@Test(dependsOnMethods = "testAddSpec")
public void testUpdateSpec() throws Exception {
// update is only treated as an update for existing job specs
String updatedSpecUriString = "/foo/bar/addedSpec";
Spec spec = initJobSpec(updatedSpecUriString);
WriteResponse writeResponse = (WriteResponse) _seip.updateSpec(spec).get();
log.info("WriteResponse: " + writeResponse);
List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.UPDATE), "Verb did not match");
Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(updatedSpecUriString), "Expected URI did not match");
Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}
use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class SimpleKafkaSpecExecutorTest method testAddSpec.
@Test
public void testAddSpec() throws Exception {
_closer = Closer.create();
_properties = new Properties();
// Properties for Producer
_properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, TOPIC);
_properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", _kafkaBrokers);
_properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
// Properties for Consumer
_properties.setProperty(ConfigurationKeys.KAFKA_BROKERS, _kafkaBrokers);
_properties.setProperty(SimpleKafkaSpecExecutor.SPEC_KAFKA_TOPICS_KEY, TOPIC);
// SEI Producer
_seip = _closer.register(new SimpleKafkaSpecProducer(ConfigUtils.propertiesToConfig(_properties)));
String addedSpecUriString = "/foo/bar/addedSpec";
Spec spec = initJobSpec(addedSpecUriString);
WriteResponse writeResponse = (WriteResponse) _seip.addSpec(spec).get();
log.info("WriteResponse: " + writeResponse);
try {
Thread.sleep(1000);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
_seic = _closer.register(new SimpleKafkaSpecConsumer(ConfigUtils.propertiesToConfig(_properties)));
List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.ADD), "Verb did not match");
Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(addedSpecUriString), "Expected URI did not match");
Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}
use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class SimpleKafkaSpecConsumer method changedSpecs.
@Override
public Future<? extends List<Pair<SpecExecutor.Verb, Spec>>> changedSpecs() {
List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = new ArrayList<>();
initializeWatermarks();
this.currentPartitionIdx = -1;
while (!allPartitionsFinished()) {
if (currentPartitionFinished()) {
moveToNextPartition();
continue;
}
if (this.messageIterator == null || !this.messageIterator.hasNext()) {
try {
this.messageIterator = fetchNextMessageBuffer();
} catch (Exception e) {
log.error(String.format("Failed to fetch next message buffer for partition %s. Will skip this partition.", getCurrentPartition()), e);
moveToNextPartition();
continue;
}
if (this.messageIterator == null || !this.messageIterator.hasNext()) {
moveToNextPartition();
continue;
}
}
while (!currentPartitionFinished()) {
if (!this.messageIterator.hasNext()) {
break;
}
KafkaConsumerRecord nextValidMessage = this.messageIterator.next();
// until we get to x.
if (nextValidMessage.getOffset() < _nextWatermark.get(this.currentPartitionIdx)) {
continue;
}
_nextWatermark.set(this.currentPartitionIdx, nextValidMessage.getNextOffset());
try {
final AvroJobSpec record;
if (nextValidMessage instanceof ByteArrayBasedKafkaRecord) {
record = decodeRecord((ByteArrayBasedKafkaRecord) nextValidMessage);
} else if (nextValidMessage instanceof DecodeableKafkaRecord) {
record = ((DecodeableKafkaRecord<?, AvroJobSpec>) nextValidMessage).getValue();
} else {
throw new IllegalStateException("Unsupported KafkaConsumerRecord type. The returned record can either be ByteArrayBasedKafkaRecord" + " or DecodeableKafkaRecord");
}
JobSpec.Builder jobSpecBuilder = JobSpec.builder(record.getUri());
Properties props = new Properties();
props.putAll(record.getProperties());
jobSpecBuilder.withJobCatalogURI(record.getUri()).withVersion(record.getVersion()).withDescription(record.getDescription()).withConfigAsProperties(props);
if (!record.getTemplateUri().isEmpty()) {
jobSpecBuilder.withTemplate(new URI(record.getTemplateUri()));
}
String verbName = record.getMetadata().get(VERB_KEY);
SpecExecutor.Verb verb = SpecExecutor.Verb.valueOf(verbName);
changesSpecs.add(new ImmutablePair<SpecExecutor.Verb, Spec>(verb, jobSpecBuilder.build()));
} catch (Throwable t) {
log.error("Could not decode record at partition " + this.currentPartitionIdx + " offset " + nextValidMessage.getOffset());
}
}
}
return new CompletedFuture(changesSpecs, null);
}
use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class FlowCatalog method addListener.
@Override
public void addListener(SpecCatalogListener specListener) {
Preconditions.checkNotNull(specListener);
this.listeners.addListener(specListener);
if (state() == State.RUNNING) {
for (Spec spec : getSpecsWithTimeUpdate()) {
SpecCatalogListener.AddSpecCallback addJobCallback = new SpecCatalogListener.AddSpecCallback(spec);
this.listeners.callbackOneListener(addJobCallback, specListener);
}
}
}
Aggregations