Examples with SpecExecutor org.apache.gobblin.runtime.api.SpecExecutor used on opensource projects

Example 1 with SpecExecutor

use of org.apache.gobblin.runtime.api.SpecExecutor in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompilerTest method testCompilerWithoutTemplateCatalog.

@Test
public void testCompilerWithoutTemplateCatalog() {
    FlowSpec flowSpec = initFlowSpec();
    // Run compiler on flowSpec
    Map<Spec, SpecExecutor> specExecutorMapping = this.compilerWithoutTemplateCalague.compileFlow(flowSpec);
    // Assert pre-requisites
    Assert.assertNotNull(specExecutorMapping, "Expected non null mapping.");
    Assert.assertTrue(specExecutorMapping.size() == 1, "Exepected 1 executor for FlowSpec.");
    // Assert FlowSpec compilation
    Spec spec = specExecutorMapping.keySet().iterator().next();
    Assert.assertTrue(spec instanceof JobSpec, "Expected JobSpec compiled from FlowSpec.");
    // Assert JobSpec properties
    JobSpec jobSpec = (JobSpec) spec;
    Assert.assertTrue(!jobSpec.getConfig().hasPath("testProperty1"));
    Assert.assertTrue(!jobSpec.getConfig().hasPath("testProperty2"));
    Assert.assertTrue(!jobSpec.getConfig().hasPath("testProperty3"));
    Assert.assertEquals(jobSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY), TEST_SOURCE_NAME);
    Assert.assertFalse(jobSpec.getConfig().hasPath(ConfigurationKeys.JOB_SCHEDULE_KEY));
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.JOB_NAME_KEY), TEST_FLOW_NAME);
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.JOB_GROUP_KEY), TEST_FLOW_GROUP);
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), TEST_FLOW_NAME);
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), TEST_FLOW_GROUP);
    Assert.assertTrue(jobSpec.getConfig().hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY));
}

Example 2 with SpecExecutor

use of org.apache.gobblin.runtime.api.SpecExecutor in project incubator-gobblin by apache.

the class Orchestrator method orchestrate.

public void orchestrate(Spec spec) throws Exception {
    // Add below waiting because TopologyCatalog and FlowCatalog service can be launched at the same time
    this.topologyCatalog.get().getInitComplete().await();
    long startTime = System.nanoTime();
    if (spec instanceof FlowSpec) {
        Map<Spec, SpecExecutor> specExecutorInstanceMap = specCompiler.compileFlow(spec);
        if (specExecutorInstanceMap.isEmpty()) {
            _log.warn("Cannot determine an executor to run on for Spec: " + spec);
            return;
        }
        // Schedule all compiled JobSpecs on their respective Executor
        for (Map.Entry<Spec, SpecExecutor> specsToExecute : specExecutorInstanceMap.entrySet()) {
            // Run this spec on selected executor
            SpecProducer producer = null;
            try {
                producer = specsToExecute.getValue().getProducer().get();
                Spec jobSpec = specsToExecute.getKey();
                _log.info(String.format("Going to orchestrate JobSpec: %s on Executor: %s", jobSpec, producer));
                producer.addSpec(jobSpec);
            } catch (Exception e) {
                _log.error("Cannot successfully setup spec: " + specsToExecute.getKey() + " on executor: " + producer + " for flow: " + spec, e);
            }
        }
    } else {
        Instrumented.markMeter(this.flowOrchestrationFailedMeter);
        throw new RuntimeException("Spec not of type FlowSpec, cannot orchestrate: " + spec);
    }
    Instrumented.markMeter(this.flowOrchestrationSuccessFulMeter);
    Instrumented.updateTimer(this.flowOrchestrationTimer, System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
}

Example 3 with SpecExecutor

use of org.apache.gobblin.runtime.api.SpecExecutor in project incubator-gobblin by apache.

the class Orchestrator method remove.

public void remove(Spec spec) {
    // Note: Current logic assumes compilation is consistent between all executions
    if (spec instanceof FlowSpec) {
        Map<Spec, SpecExecutor> specExecutorInstanceMap = specCompiler.compileFlow(spec);
        if (specExecutorInstanceMap.isEmpty()) {
            _log.warn("Cannot determine an executor to delete Spec: " + spec);
            return;
        }
        // Delete all compiled JobSpecs on their respective Executor
        for (Map.Entry<Spec, SpecExecutor> specsToDelete : specExecutorInstanceMap.entrySet()) {
            // Delete this spec on selected executor
            SpecProducer producer = null;
            try {
                producer = specsToDelete.getValue().getProducer().get();
                Spec jobSpec = specsToDelete.getKey();
                _log.info(String.format("Going to delete JobSpec: %s on Executor: %s", jobSpec, producer));
                producer.deleteSpec(jobSpec.getUri());
            } catch (Exception e) {
                _log.error("Cannot successfully delete spec: " + specsToDelete.getKey() + " on executor: " + producer + " for flow: " + spec, e);
            }
        }
    } else {
        throw new RuntimeException("Spec not of type FlowSpec, cannot delete: " + spec);
    }
}

Example 4 with SpecExecutor

use of org.apache.gobblin.runtime.api.SpecExecutor in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompiler method compileFlow.

@Override
public Map<Spec, SpecExecutor> compileFlow(Spec spec) {
    Preconditions.checkNotNull(spec);
    Preconditions.checkArgument(spec instanceof FlowSpec, "IdentityFlowToJobSpecCompiler only converts FlowSpec to JobSpec");
    long startTime = System.nanoTime();
    Map<Spec, SpecExecutor> specExecutorMap = Maps.newLinkedHashMap();
    FlowSpec flowSpec = (FlowSpec) spec;
    String source = flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY);
    String destination = flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY);
    log.info(String.format("Compiling flow for source: %s and destination: %s", source, destination));
    JobSpec jobSpec = jobSpecGenerator(flowSpec);
    for (TopologySpec topologySpec : topologySpecMap.values()) {
        try {
            Map<ServiceNode, ServiceNode> capabilities = (Map<ServiceNode, ServiceNode>) topologySpec.getSpecExecutor().getCapabilities().get();
            for (Map.Entry<ServiceNode, ServiceNode> capability : capabilities.entrySet()) {
                log.info(String.format("Evaluating current JobSpec: %s against TopologySpec: %s with " + "capability of source: %s and destination: %s ", jobSpec.getUri(), topologySpec.getUri(), capability.getKey(), capability.getValue()));
                if (source.equals(capability.getKey().getNodeName()) && destination.equals(capability.getValue().getNodeName())) {
                    specExecutorMap.put(jobSpec, topologySpec.getSpecExecutor());
                    log.info(String.format("Current JobSpec: %s is executable on TopologySpec: %s. Added TopologySpec as candidate.", jobSpec.getUri(), topologySpec.getUri()));
                    log.info("Since we found a candidate executor, we will not try to compute more. " + "(Intended limitation for IdentityFlowToJobSpecCompiler)");
                    return specExecutorMap;
                }
            }
        } catch (InterruptedException | ExecutionException e) {
            Instrumented.markMeter(this.flowCompilationFailedMeter);
            throw new RuntimeException("Cannot determine topology capabilities", e);
        }
    }
    Instrumented.markMeter(this.flowCompilationSuccessFulMeter);
    Instrumented.updateTimer(this.flowCompilationTimer, System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
    return specExecutorMap;
}

Example 5 with SpecExecutor

use of org.apache.gobblin.runtime.api.SpecExecutor in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompilerTest method testNoJobSpecCompilation.

@Test
public void testNoJobSpecCompilation() {
    FlowSpec flowSpec = initFlowSpec(TEST_FLOW_GROUP, TEST_FLOW_NAME, "unsupportedSource", "unsupportedSink");
    // Run compiler on flowSpec
    Map<Spec, SpecExecutor> specExecutorMapping = this.compilerWithTemplateCalague.compileFlow(flowSpec);
    // Assert pre-requisites
    Assert.assertNotNull(specExecutorMapping, "Expected non null mapping.");
    Assert.assertTrue(specExecutorMapping.size() == 0, "Exepected 1 executor for FlowSpec.");
}