Examples with TopologySpec org.apache.gobblin.runtime.api.TopologySpec used on opensource projects

Example 6 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompiler method compileFlow.

@Override
public Map<Spec, SpecExecutor> compileFlow(Spec spec) {
    Preconditions.checkNotNull(spec);
    Preconditions.checkArgument(spec instanceof FlowSpec, "IdentityFlowToJobSpecCompiler only converts FlowSpec to JobSpec");
    long startTime = System.nanoTime();
    Map<Spec, SpecExecutor> specExecutorMap = Maps.newLinkedHashMap();
    FlowSpec flowSpec = (FlowSpec) spec;
    String source = flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY);
    String destination = flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY);
    log.info(String.format("Compiling flow for source: %s and destination: %s", source, destination));
    JobSpec jobSpec = jobSpecGenerator(flowSpec);
    for (TopologySpec topologySpec : topologySpecMap.values()) {
        try {
            Map<ServiceNode, ServiceNode> capabilities = (Map<ServiceNode, ServiceNode>) topologySpec.getSpecExecutor().getCapabilities().get();
            for (Map.Entry<ServiceNode, ServiceNode> capability : capabilities.entrySet()) {
                log.info(String.format("Evaluating current JobSpec: %s against TopologySpec: %s with " + "capability of source: %s and destination: %s ", jobSpec.getUri(), topologySpec.getUri(), capability.getKey(), capability.getValue()));
                if (source.equals(capability.getKey().getNodeName()) && destination.equals(capability.getValue().getNodeName())) {
                    specExecutorMap.put(jobSpec, topologySpec.getSpecExecutor());
                    log.info(String.format("Current JobSpec: %s is executable on TopologySpec: %s. Added TopologySpec as candidate.", jobSpec.getUri(), topologySpec.getUri()));
                    log.info("Since we found a candidate executor, we will not try to compute more. " + "(Intended limitation for IdentityFlowToJobSpecCompiler)");
                    return specExecutorMap;
                }
            }
        } catch (InterruptedException | ExecutionException e) {
            Instrumented.markMeter(this.flowCompilationFailedMeter);
            throw new RuntimeException("Cannot determine topology capabilities", e);
        }
    }
    Instrumented.markMeter(this.flowCompilationSuccessFulMeter);
    Instrumented.updateTimer(this.flowCompilationTimer, System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
    return specExecutorMap;
}

Example 7 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompilerTest method initTopologySpec.

private TopologySpec initTopologySpec() {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", TOPOLOGY_SPEC_STORE_DIR);
    properties.put("specExecInstance.capabilities", TEST_SOURCE_NAME + ":" + TEST_SINK_NAME);
    Config config = ConfigUtils.propertiesToConfig(properties);
    SpecExecutor specExecutorInstance = new InMemorySpecExecutor(config);
    TopologySpec.Builder topologySpecBuilder = TopologySpec.builder(computeTopologySpecURI(SPEC_STORE_PARENT_DIR, TOPOLOGY_SPEC_STORE_DIR)).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withSpecExecutor(specExecutorInstance);
    return topologySpecBuilder.build();
}

Example 8 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method testWeightedGraphConstruction.

@Test
public void testWeightedGraphConstruction() {
    FlowSpec flowSpec = initFlowSpec();
    TopologySpec topologySpec = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR, TEST_SOURCE_NAME, TEST_HOP_NAME_A, TEST_HOP_NAME_B, TEST_SINK_NAME);
    this.compilerWithTemplateCalague.onAddSpec(topologySpec);
    // invocation of compileFlow trigger the weighedGraph construction
    this.compilerWithTemplateCalague.compileFlow(flowSpec);
    DirectedWeightedMultigraph<ServiceNode, FlowEdge> weightedGraph = compilerWithTemplateCalague.getWeightedGraph();
    Assert.assertTrue(weightedGraph.containsVertex(vertexSource));
    Assert.assertTrue(weightedGraph.containsVertex(vertexHopA));
    Assert.assertTrue(weightedGraph.containsVertex(vertexHopB));
    Assert.assertTrue(weightedGraph.containsVertex(vertexSink));
    FlowEdge edgeSrc2A = new LoadBasedFlowEdgeImpl(vertexSource, vertexHopA, topologySpec.getSpecExecutor());
    FlowEdge edgeA2B = new LoadBasedFlowEdgeImpl(vertexHopA, vertexHopB, topologySpec.getSpecExecutor());
    FlowEdge edgeB2Sink = new LoadBasedFlowEdgeImpl(vertexHopB, vertexSink, topologySpec.getSpecExecutor());
    Assert.assertTrue(weightedGraph.containsEdge(edgeSrc2A));
    Assert.assertTrue(weightedGraph.containsEdge(edgeA2B));
    Assert.assertTrue(weightedGraph.containsEdge(edgeB2Sink));
    Assert.assertTrue(edgeEqual(weightedGraph.getEdge(vertexSource, vertexHopA), edgeSrc2A));
    Assert.assertTrue(edgeEqual(weightedGraph.getEdge(vertexHopA, vertexHopB), edgeA2B));
    Assert.assertTrue(edgeEqual(weightedGraph.getEdge(vertexHopB, vertexSink), edgeB2Sink));
    this.compilerWithTemplateCalague.onDeleteSpec(topologySpec.getUri(), "");
}

Example 9 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method initTopologySpec.

// The topology is: Src - A - B - Dest
private TopologySpec initTopologySpec(String storeDir, String... args) {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", storeDir);
    String capabilitiesString = "";
    for (int i = 0; i < args.length - 1; i++) {
        capabilitiesString = capabilitiesString + (args[i] + ":" + args[i + 1] + ",");
    }
    Assert.assertEquals(capabilitiesString.charAt(capabilitiesString.length() - 1), ',');
    capabilitiesString = capabilitiesString.substring(0, capabilitiesString.length() - 1);
    properties.put("specExecInstance.capabilities", capabilitiesString);
    properties.put("executorAttrs", new Properties());
    Config config = ConfigUtils.propertiesToConfig(properties);
    SpecExecutor specExecutorInstance = new InMemorySpecExecutor(config);
    TopologySpec.Builder topologySpecBuilder = TopologySpec.builder(IdentityFlowToJobSpecCompilerTest.computeTopologySpecURI(SPEC_STORE_PARENT_DIR, storeDir)).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withSpecExecutor(specExecutorInstance);
    return topologySpecBuilder.build();
}

Example 10 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class OrchestratorTest method createTopologySpec.

@Test
public void createTopologySpec() {
    IdentityFlowToJobSpecCompiler specCompiler = (IdentityFlowToJobSpecCompiler) this.orchestrator.getSpecCompiler();
    // List Current Specs
    Collection<Spec> specs = topologyCatalog.getSpecs();
    logger.info("[Before Create] Number of specs: " + specs.size());
    int i = 0;
    for (Spec spec : specs) {
        TopologySpec topologySpec = (TopologySpec) spec;
        logger.info("[Before Create] Spec " + i++ + ": " + gson.toJson(topologySpec));
    }
    // Make sure TopologyCatalog is empty
    Assert.assertTrue(specs.size() == 0, "Spec store should be empty before addition");
    // Make sure TopologyCatalog Listener is empty
    Assert.assertTrue(specCompiler.getTopologySpecMap().size() == 0, "SpecCompiler should not know about any Topology " + "before addition");
    // Create and add Spec
    this.topologyCatalog.put(topologySpec);
    // List Specs after adding
    specs = topologyCatalog.getSpecs();
    logger.info("[After Create] Number of specs: " + specs.size());
    i = 0;
    for (Spec spec : specs) {
        topologySpec = (TopologySpec) spec;
        logger.info("[After Create] Spec " + i++ + ": " + gson.toJson(topologySpec));
    }
    // Make sure TopologyCatalog has the added Topology
    Assert.assertTrue(specs.size() == 1, "Spec store should contain 1 Spec after addition");
    // Make sure TopologyCatalog Listener knows about added Topology
    Assert.assertTrue(specCompiler.getTopologySpecMap().size() == 1, "SpecCompiler should contain 1 Spec after addition");
}