Examples with Spec org.apache.gobblin.runtime.api.Spec used on opensource projects

Example 11 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class Orchestrator method orchestrate.

public void orchestrate(Spec spec) throws Exception {
    // Add below waiting because TopologyCatalog and FlowCatalog service can be launched at the same time
    this.topologyCatalog.get().getInitComplete().await();
    long startTime = System.nanoTime();
    if (spec instanceof FlowSpec) {
        Map<Spec, SpecExecutor> specExecutorInstanceMap = specCompiler.compileFlow(spec);
        if (specExecutorInstanceMap.isEmpty()) {
            _log.warn("Cannot determine an executor to run on for Spec: " + spec);
            return;
        }
        // Schedule all compiled JobSpecs on their respective Executor
        for (Map.Entry<Spec, SpecExecutor> specsToExecute : specExecutorInstanceMap.entrySet()) {
            // Run this spec on selected executor
            SpecProducer producer = null;
            try {
                producer = specsToExecute.getValue().getProducer().get();
                Spec jobSpec = specsToExecute.getKey();
                _log.info(String.format("Going to orchestrate JobSpec: %s on Executor: %s", jobSpec, producer));
                producer.addSpec(jobSpec);
            } catch (Exception e) {
                _log.error("Cannot successfully setup spec: " + specsToExecute.getKey() + " on executor: " + producer + " for flow: " + spec, e);
            }
        }
    } else {
        Instrumented.markMeter(this.flowOrchestrationFailedMeter);
        throw new RuntimeException("Spec not of type FlowSpec, cannot orchestrate: " + spec);
    }
    Instrumented.markMeter(this.flowOrchestrationSuccessFulMeter);
    Instrumented.updateTimer(this.flowOrchestrationTimer, System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
}

Example 12 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class Orchestrator method remove.

public void remove(Spec spec) {
    // Note: Current logic assumes compilation is consistent between all executions
    if (spec instanceof FlowSpec) {
        Map<Spec, SpecExecutor> specExecutorInstanceMap = specCompiler.compileFlow(spec);
        if (specExecutorInstanceMap.isEmpty()) {
            _log.warn("Cannot determine an executor to delete Spec: " + spec);
            return;
        }
        // Delete all compiled JobSpecs on their respective Executor
        for (Map.Entry<Spec, SpecExecutor> specsToDelete : specExecutorInstanceMap.entrySet()) {
            // Delete this spec on selected executor
            SpecProducer producer = null;
            try {
                producer = specsToDelete.getValue().getProducer().get();
                Spec jobSpec = specsToDelete.getKey();
                _log.info(String.format("Going to delete JobSpec: %s on Executor: %s", jobSpec, producer));
                producer.deleteSpec(jobSpec.getUri());
            } catch (Exception e) {
                _log.error("Cannot successfully delete spec: " + specsToDelete.getKey() + " on executor: " + producer + " for flow: " + spec, e);
            }
        }
    } else {
        throw new RuntimeException("Spec not of type FlowSpec, cannot delete: " + spec);
    }
}

Example 13 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class GobblinServiceJobScheduler method runJob.

@Override
public void runJob(Properties jobProps, JobListener jobListener) throws JobException {
    try {
        Spec flowSpec = this.scheduledFlowSpecs.get(jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY));
        this.orchestrator.orchestrate(flowSpec);
    } catch (Exception e) {
        throw new JobException("Failed to run Spec: " + jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY), e);
    }
}

Example 14 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class FlowCatalogTest method createFlowSpec.

@Test
public void createFlowSpec() {
    // List Current Specs
    Collection<Spec> specs = flowCatalog.getSpecs();
    logger.info("[Before Create] Number of specs: " + specs.size());
    int i = 0;
    for (Spec spec : specs) {
        FlowSpec flowSpec = (FlowSpec) spec;
        logger.info("[Before Create] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    Assert.assertTrue(specs.size() == 0, "Spec store should be empty before addition");
    // Create and add Spec
    this.flowCatalog.put(flowSpec);
    // List Specs after adding
    specs = flowCatalog.getSpecs();
    logger.info("[After Create] Number of specs: " + specs.size());
    i = 0;
    for (Spec spec : specs) {
        flowSpec = (FlowSpec) spec;
        logger.info("[After Create] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    Assert.assertTrue(specs.size() == 1, "Spec store should contain 1 Spec after addition");
}

Example 15 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class TopologyCatalogTest method deleteTopologySpec.

@Test(dependsOnMethods = "createTopologySpec")
public void deleteTopologySpec() {
    // List Current Specs
    Collection<Spec> specs = topologyCatalog.getSpecs();
    logger.info("[Before Delete] Number of specs: " + specs.size());
    int i = 0;
    for (Spec spec : specs) {
        TopologySpec topologySpec = (TopologySpec) spec;
        logger.info("[Before Delete] Spec " + i++ + ": " + gson.toJson(topologySpec));
    }
    Assert.assertTrue(specs.size() == 1, "Spec store should initially have 1 Spec before deletion");
    this.topologyCatalog.remove(topologySpec.getUri());
    // List Specs after adding
    specs = topologyCatalog.getSpecs();
    logger.info("[After Create] Number of specs: " + specs.size());
    i = 0;
    for (Spec spec : specs) {
        topologySpec = (TopologySpec) spec;
        logger.info("[After Create] Spec " + i++ + ": " + gson.toJson(topologySpec));
    }
    Assert.assertTrue(specs.size() == 0, "Spec store should be empty after deletion");
}