use of org.apache.gobblin.runtime.api.SpecProducer in project incubator-gobblin by apache.
the class Orchestrator method orchestrate.
public void orchestrate(Spec spec) throws Exception {
// Add below waiting because TopologyCatalog and FlowCatalog service can be launched at the same time
this.topologyCatalog.get().getInitComplete().await();
long startTime = System.nanoTime();
if (spec instanceof FlowSpec) {
Map<Spec, SpecExecutor> specExecutorInstanceMap = specCompiler.compileFlow(spec);
if (specExecutorInstanceMap.isEmpty()) {
_log.warn("Cannot determine an executor to run on for Spec: " + spec);
return;
}
// Schedule all compiled JobSpecs on their respective Executor
for (Map.Entry<Spec, SpecExecutor> specsToExecute : specExecutorInstanceMap.entrySet()) {
// Run this spec on selected executor
SpecProducer producer = null;
try {
producer = specsToExecute.getValue().getProducer().get();
Spec jobSpec = specsToExecute.getKey();
_log.info(String.format("Going to orchestrate JobSpec: %s on Executor: %s", jobSpec, producer));
producer.addSpec(jobSpec);
} catch (Exception e) {
_log.error("Cannot successfully setup spec: " + specsToExecute.getKey() + " on executor: " + producer + " for flow: " + spec, e);
}
}
} else {
Instrumented.markMeter(this.flowOrchestrationFailedMeter);
throw new RuntimeException("Spec not of type FlowSpec, cannot orchestrate: " + spec);
}
Instrumented.markMeter(this.flowOrchestrationSuccessFulMeter);
Instrumented.updateTimer(this.flowOrchestrationTimer, System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
}
use of org.apache.gobblin.runtime.api.SpecProducer in project incubator-gobblin by apache.
the class Orchestrator method remove.
public void remove(Spec spec) {
// Note: Current logic assumes compilation is consistent between all executions
if (spec instanceof FlowSpec) {
Map<Spec, SpecExecutor> specExecutorInstanceMap = specCompiler.compileFlow(spec);
if (specExecutorInstanceMap.isEmpty()) {
_log.warn("Cannot determine an executor to delete Spec: " + spec);
return;
}
// Delete all compiled JobSpecs on their respective Executor
for (Map.Entry<Spec, SpecExecutor> specsToDelete : specExecutorInstanceMap.entrySet()) {
// Delete this spec on selected executor
SpecProducer producer = null;
try {
producer = specsToDelete.getValue().getProducer().get();
Spec jobSpec = specsToDelete.getKey();
_log.info(String.format("Going to delete JobSpec: %s on Executor: %s", jobSpec, producer));
producer.deleteSpec(jobSpec.getUri());
} catch (Exception e) {
_log.error("Cannot successfully delete spec: " + specsToDelete.getKey() + " on executor: " + producer + " for flow: " + spec, e);
}
}
} else {
throw new RuntimeException("Spec not of type FlowSpec, cannot delete: " + spec);
}
}
Aggregations