Examples with TopologySpec org.apache.gobblin.runtime.api.TopologySpec used on opensource projects

Example 1 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method testDijkstraPathFinding.

@Test(dependsOnMethods = "testWeightedGraphConstruction")
public void testDijkstraPathFinding() {
    FlowSpec flowSpec = initFlowSpec();
    TopologySpec topologySpec_1 = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR, TEST_SOURCE_NAME, TEST_HOP_NAME_A, TEST_HOP_NAME_B, TEST_SINK_NAME);
    TopologySpec topologySpec_2 = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR_SECOND, TEST_SOURCE_NAME, TEST_HOP_NAME_B, TEST_HOP_NAME_C, TEST_SINK_NAME);
    this.compilerWithTemplateCalague.onAddSpec(topologySpec_1);
    this.compilerWithTemplateCalague.onAddSpec(topologySpec_2);
    // Get the edge -> Change the weight -> Materialized the edge change back to graph -> compile again -> Assertion
    this.compilerWithTemplateCalague.compileFlow(flowSpec);
    DirectedWeightedMultigraph<ServiceNode, FlowEdge> weightedGraph = compilerWithTemplateCalague.getWeightedGraph();
    FlowEdge a2b = weightedGraph.getEdge(vertexHopA, vertexHopB);
    FlowEdge b2c = weightedGraph.getEdge(vertexHopB, vertexHopC);
    FlowEdge c2s = weightedGraph.getEdge(vertexHopC, vertexSink);
    weightedGraph.setEdgeWeight(a2b, 1.99);
    weightedGraph.setEdgeWeight(b2c, 0.1);
    weightedGraph.setEdgeWeight(c2s, 0.2);
    // Best route: Src - B(1) - C(0.1) - sink (0.2)
    this.compilerWithTemplateCalague.compileFlow(flowSpec);
    List<FlowEdge> edgeList = dijkstraBasedPathFindingHelper(vertexSource, vertexSink, weightedGraph);
    FlowEdge src2b = weightedGraph.getEdge(vertexSource, vertexHopB);
    FlowEdge b2C = weightedGraph.getEdge(vertexHopB, vertexHopC);
    FlowEdge c2sink = weightedGraph.getEdge(vertexHopC, vertexSink);
    Assert.assertEquals(edgeList.get(0).getEdgeIdentity(), src2b.getEdgeIdentity());
    Assert.assertEquals(edgeList.get(1).getEdgeIdentity(), b2C.getEdgeIdentity());
    Assert.assertEquals(edgeList.get(2).getEdgeIdentity(), c2sink.getEdgeIdentity());
    this.compilerWithTemplateCalague.onDeleteSpec(topologySpec_1.getUri(), "");
    this.compilerWithTemplateCalague.onDeleteSpec(topologySpec_2.getUri(), "");
}

Example 2 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method testServicePolicy.

@Test
public void testServicePolicy() {
    // Initialize compiler with some blacklist properties
    Properties properties = new Properties();
    properties.setProperty(ServiceConfigKeys.TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY, TEST_TEMPLATE_CATALOG_URI);
    String testPath = TEST_SOURCE_NAME + "," + TEST_HOP_NAME_A + "," + TEST_HOP_NAME_B + "," + TEST_SINK_NAME;
    properties.setProperty(ServiceConfigKeys.POLICY_BASED_DATA_MOVEMENT_PATH, testPath);
    properties.setProperty(ServiceConfigKeys.POLICY_BASED_BLOCKED_NODES, "testHopA");
    MultiHopsFlowToJobSpecCompiler compiler = new MultiHopsFlowToJobSpecCompiler(ConfigUtils.propertiesToConfig(properties));
    FlowSpec flowSpec = initFlowSpec();
    TopologySpec topologySpec = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR, TEST_SOURCE_NAME, TEST_HOP_NAME_A, TEST_HOP_NAME_B, TEST_SINK_NAME);
    compiler.onAddSpec(topologySpec);
    // invocation of compileFlow trigger the weighedGraph construction
    compiler.compileFlow(flowSpec);
    compiler.servicePolicy.populateBlackListedEdges(compiler.getWeightedGraph());
    Assert.assertEquals(compiler.servicePolicy.getBlacklistedEdges().size(), 2);
    FlowEdge edgeSrc2A = new LoadBasedFlowEdgeImpl(vertexSource, vertexHopA, topologySpec.getSpecExecutor());
    FlowEdge edgeA2B = new LoadBasedFlowEdgeImpl(vertexHopA, vertexHopB, topologySpec.getSpecExecutor());
    Assert.assertTrue(compiler.servicePolicy.getBlacklistedEdges().contains(edgeSrc2A));
    Assert.assertTrue(compiler.servicePolicy.getBlacklistedEdges().contains(edgeA2B));
}

Example 3 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class TopologyCatalogTest method deleteTopologySpec.

@Test(dependsOnMethods = "createTopologySpec")
public void deleteTopologySpec() {
    // List Current Specs
    Collection<Spec> specs = topologyCatalog.getSpecs();
    logger.info("[Before Delete] Number of specs: " + specs.size());
    int i = 0;
    for (Spec spec : specs) {
        TopologySpec topologySpec = (TopologySpec) spec;
        logger.info("[Before Delete] Spec " + i++ + ": " + gson.toJson(topologySpec));
    }
    Assert.assertTrue(specs.size() == 1, "Spec store should initially have 1 Spec before deletion");
    this.topologyCatalog.remove(topologySpec.getUri());
    // List Specs after adding
    specs = topologyCatalog.getSpecs();
    logger.info("[After Create] Number of specs: " + specs.size());
    i = 0;
    for (Spec spec : specs) {
        topologySpec = (TopologySpec) spec;
        logger.info("[After Create] Spec " + i++ + ": " + gson.toJson(topologySpec));
    }
    Assert.assertTrue(specs.size() == 0, "Spec store should be empty after deletion");
}

Example 4 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class GobblinServiceManager method start.

@Override
public void start() throws ApplicationException {
    LOGGER.info("[Init] Starting the Gobblin Service Manager");
    if (this.helixManager.isPresent()) {
        connectHelixManager();
    }
    this.eventBus.register(this);
    this.serviceLauncher.start();
    if (this.helixManager.isPresent()) {
        // Subscribe to leadership changes
        this.helixManager.get().addControllerListener(new ControllerChangeListener() {

            @Override
            public void onControllerChange(NotificationContext changeContext) {
                handleLeadershipChange(changeContext);
            }
        });
        // Update for first time since there might be no notification
        if (helixManager.get().isLeader()) {
            if (this.isSchedulerEnabled) {
                LOGGER.info("[Init] Gobblin Service is running in master instance mode, enabling Scheduler.");
                this.scheduler.setActive(true);
            }
            if (this.isGitConfigMonitorEnabled) {
                this.gitConfigMonitor.setActive(true);
            }
        } else {
            if (this.isSchedulerEnabled) {
                LOGGER.info("[Init] Gobblin Service is running in slave instance mode, not enabling Scheduler.");
            }
        }
    } else {
        // No Helix manager, hence standalone service instance
        // .. designate scheduler to itself
        LOGGER.info("[Init] Gobblin Service is running in single instance mode, enabling Scheduler.");
        this.scheduler.setActive(true);
        if (this.isGitConfigMonitorEnabled) {
            this.gitConfigMonitor.setActive(true);
        }
    }
    // This has to be done after the topologyCatalog service is launched
    if (this.isTopologySpecFactoryEnabled) {
        Collection<TopologySpec> topologySpecs = this.topologySpecFactory.getTopologies();
        for (TopologySpec topologySpec : topologySpecs) {
            this.topologyCatalog.put(topologySpec);
        }
    }
    // This has to be done after topologySpecFactory has updated spec store, so that listeners will have the latest updates.
    if (isSchedulerEnabled) {
        this.topologyCatalog.addListener(this.orchestrator);
    }
    // Notify now topologyCatalog has the right information
    this.topologyCatalog.getInitComplete().countDown();
}

Example 5 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class BaseFlowToJobSpecCompiler method onAddSpec.

@Override
public synchronized void onAddSpec(Spec addedSpec) {
    TopologySpec spec = (TopologySpec) addedSpec;
    log.info("Loading topology {}", spec.toLongString());
    for (Map.Entry entry : spec.getConfigAsProperties().entrySet()) {
        log.info("topo: {} --> {}", entry.getKey(), entry.getValue());
    }
    topologySpecMap.put(addedSpec.getUri(), (TopologySpec) addedSpec);
}