Examples with FlowSpec org.apache.gobblin.runtime.api.FlowSpec used on opensource projects

Example 6 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method testServicePolicy.

@Test
public void testServicePolicy() {
    // Initialize compiler with some blacklist properties
    Properties properties = new Properties();
    properties.setProperty(ServiceConfigKeys.TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY, TEST_TEMPLATE_CATALOG_URI);
    String testPath = TEST_SOURCE_NAME + "," + TEST_HOP_NAME_A + "," + TEST_HOP_NAME_B + "," + TEST_SINK_NAME;
    properties.setProperty(ServiceConfigKeys.POLICY_BASED_DATA_MOVEMENT_PATH, testPath);
    properties.setProperty(ServiceConfigKeys.POLICY_BASED_BLOCKED_NODES, "testHopA");
    MultiHopsFlowToJobSpecCompiler compiler = new MultiHopsFlowToJobSpecCompiler(ConfigUtils.propertiesToConfig(properties));
    FlowSpec flowSpec = initFlowSpec();
    TopologySpec topologySpec = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR, TEST_SOURCE_NAME, TEST_HOP_NAME_A, TEST_HOP_NAME_B, TEST_SINK_NAME);
    compiler.onAddSpec(topologySpec);
    // invocation of compileFlow trigger the weighedGraph construction
    compiler.compileFlow(flowSpec);
    compiler.servicePolicy.populateBlackListedEdges(compiler.getWeightedGraph());
    Assert.assertEquals(compiler.servicePolicy.getBlacklistedEdges().size(), 2);
    FlowEdge edgeSrc2A = new LoadBasedFlowEdgeImpl(vertexSource, vertexHopA, topologySpec.getSpecExecutor());
    FlowEdge edgeA2B = new LoadBasedFlowEdgeImpl(vertexHopA, vertexHopB, topologySpec.getSpecExecutor());
    Assert.assertTrue(compiler.servicePolicy.getBlacklistedEdges().contains(edgeSrc2A));
    Assert.assertTrue(compiler.servicePolicy.getBlacklistedEdges().contains(edgeA2B));
}

Example 7 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class Orchestrator method orchestrate.

public void orchestrate(Spec spec) throws Exception {
    // Add below waiting because TopologyCatalog and FlowCatalog service can be launched at the same time
    this.topologyCatalog.get().getInitComplete().await();
    long startTime = System.nanoTime();
    if (spec instanceof FlowSpec) {
        Map<Spec, SpecExecutor> specExecutorInstanceMap = specCompiler.compileFlow(spec);
        if (specExecutorInstanceMap.isEmpty()) {
            _log.warn("Cannot determine an executor to run on for Spec: " + spec);
            return;
        }
        // Schedule all compiled JobSpecs on their respective Executor
        for (Map.Entry<Spec, SpecExecutor> specsToExecute : specExecutorInstanceMap.entrySet()) {
            // Run this spec on selected executor
            SpecProducer producer = null;
            try {
                producer = specsToExecute.getValue().getProducer().get();
                Spec jobSpec = specsToExecute.getKey();
                _log.info(String.format("Going to orchestrate JobSpec: %s on Executor: %s", jobSpec, producer));
                producer.addSpec(jobSpec);
            } catch (Exception e) {
                _log.error("Cannot successfully setup spec: " + specsToExecute.getKey() + " on executor: " + producer + " for flow: " + spec, e);
            }
        }
    } else {
        Instrumented.markMeter(this.flowOrchestrationFailedMeter);
        throw new RuntimeException("Spec not of type FlowSpec, cannot orchestrate: " + spec);
    }
    Instrumented.markMeter(this.flowOrchestrationSuccessFulMeter);
    Instrumented.updateTimer(this.flowOrchestrationTimer, System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
}

Example 8 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class Orchestrator method remove.

public void remove(Spec spec) {
    // Note: Current logic assumes compilation is consistent between all executions
    if (spec instanceof FlowSpec) {
        Map<Spec, SpecExecutor> specExecutorInstanceMap = specCompiler.compileFlow(spec);
        if (specExecutorInstanceMap.isEmpty()) {
            _log.warn("Cannot determine an executor to delete Spec: " + spec);
            return;
        }
        // Delete all compiled JobSpecs on their respective Executor
        for (Map.Entry<Spec, SpecExecutor> specsToDelete : specExecutorInstanceMap.entrySet()) {
            // Delete this spec on selected executor
            SpecProducer producer = null;
            try {
                producer = specsToDelete.getValue().getProducer().get();
                Spec jobSpec = specsToDelete.getKey();
                _log.info(String.format("Going to delete JobSpec: %s on Executor: %s", jobSpec, producer));
                producer.deleteSpec(jobSpec.getUri());
            } catch (Exception e) {
                _log.error("Cannot successfully delete spec: " + specsToDelete.getKey() + " on executor: " + producer + " for flow: " + spec, e);
            }
        }
    } else {
        throw new RuntimeException("Spec not of type FlowSpec, cannot delete: " + spec);
    }
}

Example 9 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class GobblinServiceJobScheduler method disableFlowRunImmediatelyOnStart.

@VisibleForTesting
protected static Spec disableFlowRunImmediatelyOnStart(FlowSpec spec) {
    Properties properties = spec.getConfigAsProperties();
    properties.setProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "false");
    Config config = ConfigFactory.parseProperties(properties);
    FlowSpec flowSpec = new FlowSpec(spec.getUri(), spec.getVersion(), spec.getDescription(), config, properties, spec.getTemplateURIs(), spec.getChildSpecs());
    return flowSpec;
}

Example 10 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class GobblinServiceJobScheduler method runJob.

@Override
public void runJob(Properties jobProps, JobListener jobListener) throws JobException {
    try {
        Spec flowSpec = this.scheduledFlowSpecs.get(jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY));
        this.orchestrator.orchestrate(flowSpec);
    } catch (Exception e) {
        throw new JobException("Failed to run Spec: " + jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY), e);
    }
}