Examples with FlowEdge org.apache.gobblin.runtime.api.FlowEdge used on opensource projects

Example 1 with FlowEdge

use of org.apache.gobblin.runtime.api.FlowEdge in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method testDijkstraPathFinding.

@Test(dependsOnMethods = "testWeightedGraphConstruction")
public void testDijkstraPathFinding() {
    FlowSpec flowSpec = initFlowSpec();
    TopologySpec topologySpec_1 = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR, TEST_SOURCE_NAME, TEST_HOP_NAME_A, TEST_HOP_NAME_B, TEST_SINK_NAME);
    TopologySpec topologySpec_2 = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR_SECOND, TEST_SOURCE_NAME, TEST_HOP_NAME_B, TEST_HOP_NAME_C, TEST_SINK_NAME);
    this.compilerWithTemplateCalague.onAddSpec(topologySpec_1);
    this.compilerWithTemplateCalague.onAddSpec(topologySpec_2);
    // Get the edge -> Change the weight -> Materialized the edge change back to graph -> compile again -> Assertion
    this.compilerWithTemplateCalague.compileFlow(flowSpec);
    DirectedWeightedMultigraph<ServiceNode, FlowEdge> weightedGraph = compilerWithTemplateCalague.getWeightedGraph();
    FlowEdge a2b = weightedGraph.getEdge(vertexHopA, vertexHopB);
    FlowEdge b2c = weightedGraph.getEdge(vertexHopB, vertexHopC);
    FlowEdge c2s = weightedGraph.getEdge(vertexHopC, vertexSink);
    weightedGraph.setEdgeWeight(a2b, 1.99);
    weightedGraph.setEdgeWeight(b2c, 0.1);
    weightedGraph.setEdgeWeight(c2s, 0.2);
    // Best route: Src - B(1) - C(0.1) - sink (0.2)
    this.compilerWithTemplateCalague.compileFlow(flowSpec);
    List<FlowEdge> edgeList = dijkstraBasedPathFindingHelper(vertexSource, vertexSink, weightedGraph);
    FlowEdge src2b = weightedGraph.getEdge(vertexSource, vertexHopB);
    FlowEdge b2C = weightedGraph.getEdge(vertexHopB, vertexHopC);
    FlowEdge c2sink = weightedGraph.getEdge(vertexHopC, vertexSink);
    Assert.assertEquals(edgeList.get(0).getEdgeIdentity(), src2b.getEdgeIdentity());
    Assert.assertEquals(edgeList.get(1).getEdgeIdentity(), b2C.getEdgeIdentity());
    Assert.assertEquals(edgeList.get(2).getEdgeIdentity(), c2sink.getEdgeIdentity());
    this.compilerWithTemplateCalague.onDeleteSpec(topologySpec_1.getUri(), "");
    this.compilerWithTemplateCalague.onDeleteSpec(topologySpec_2.getUri(), "");
}

Example 2 with FlowEdge

use of org.apache.gobblin.runtime.api.FlowEdge in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method testServicePolicy.

@Test
public void testServicePolicy() {
    // Initialize compiler with some blacklist properties
    Properties properties = new Properties();
    properties.setProperty(ServiceConfigKeys.TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY, TEST_TEMPLATE_CATALOG_URI);
    String testPath = TEST_SOURCE_NAME + "," + TEST_HOP_NAME_A + "," + TEST_HOP_NAME_B + "," + TEST_SINK_NAME;
    properties.setProperty(ServiceConfigKeys.POLICY_BASED_DATA_MOVEMENT_PATH, testPath);
    properties.setProperty(ServiceConfigKeys.POLICY_BASED_BLOCKED_NODES, "testHopA");
    MultiHopsFlowToJobSpecCompiler compiler = new MultiHopsFlowToJobSpecCompiler(ConfigUtils.propertiesToConfig(properties));
    FlowSpec flowSpec = initFlowSpec();
    TopologySpec topologySpec = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR, TEST_SOURCE_NAME, TEST_HOP_NAME_A, TEST_HOP_NAME_B, TEST_SINK_NAME);
    compiler.onAddSpec(topologySpec);
    // invocation of compileFlow trigger the weighedGraph construction
    compiler.compileFlow(flowSpec);
    compiler.servicePolicy.populateBlackListedEdges(compiler.getWeightedGraph());
    Assert.assertEquals(compiler.servicePolicy.getBlacklistedEdges().size(), 2);
    FlowEdge edgeSrc2A = new LoadBasedFlowEdgeImpl(vertexSource, vertexHopA, topologySpec.getSpecExecutor());
    FlowEdge edgeA2B = new LoadBasedFlowEdgeImpl(vertexHopA, vertexHopB, topologySpec.getSpecExecutor());
    Assert.assertTrue(compiler.servicePolicy.getBlacklistedEdges().contains(edgeSrc2A));
    Assert.assertTrue(compiler.servicePolicy.getBlacklistedEdges().contains(edgeA2B));
}

Example 3 with FlowEdge

use of org.apache.gobblin.runtime.api.FlowEdge in project incubator-gobblin by apache.

the class FindPathUtils method dijkstraBasedPathFindingHelper.

// Since Author{autumnust@gmail.com} couldn't find the proper way to conduct Library provided by JGraphT
// on the customized-edge Graph, here is the raw implementation of Dijkstra algorithm for finding shortest path.
/**
 * Given sourceNode and targetNode, find the shortest path and return shortest path.
 * @return Each edge on this shortest path, in order.
 */
@VisibleForTesting
public static List<FlowEdge> dijkstraBasedPathFindingHelper(ServiceNode sourceNode, ServiceNode targetNode, DirectedWeightedMultigraph<ServiceNode, FlowEdge> weightedGraph) {
    Map<DistancedNode, ArrayList<FlowEdge>> shortestPath = new HashMap<>();
    Map<DistancedNode, Double> shortestDist = new HashMap<>();
    PriorityQueue<DistancedNode> pq = new PriorityQueue<>(new Comparator<DistancedNode>() {

        @Override
        public int compare(DistancedNode o1, DistancedNode o2) {
            if (o1.getDistToSrc() < o2.getDistToSrc()) {
                return -1;
            } else {
                return 1;
            }
        }
    });
    pq.add(new DistancedNode(sourceNode, 0.0));
    Set<FlowEdge> visitedEdge = new HashSet<>();
    while (!pq.isEmpty()) {
        DistancedNode node = pq.poll();
        if (node.getNode().getNodeName().equals(targetNode.getNodeName())) {
            // Searching finished
            return shortestPath.get(node);
        }
        Set<FlowEdge> outgoingEdges = weightedGraph.outgoingEdgesOf(node.getNode());
        for (FlowEdge outGoingEdge : outgoingEdges) {
            // Since it is a multi-graph problem, should use edge for deduplicaiton instead of vertex.
            if (visitedEdge.contains(outGoingEdge)) {
                continue;
            }
            DistancedNode adjacentNode = new DistancedNode(weightedGraph.getEdgeTarget(outGoingEdge));
            if (shortestDist.containsKey(adjacentNode)) {
                adjacentNode.setDistToSrc(shortestDist.get(adjacentNode));
            }
            double newDist = node.getDistToSrc() + ((LoadBasedFlowEdgeImpl) outGoingEdge).getEdgeLoad();
            if (newDist < adjacentNode.getDistToSrc()) {
                if (pq.contains(adjacentNode)) {
                    pq.remove(adjacentNode);
                }
                // Update the shortest path.
                ArrayList<FlowEdge> path = shortestPath.containsKey(node) ? new ArrayList<>(shortestPath.get(node)) : new ArrayList<>();
                path.add(outGoingEdge);
                shortestPath.put(adjacentNode, path);
                shortestDist.put(adjacentNode, newDist);
                adjacentNode.setDistToSrc(newDist);
                pq.add(adjacentNode);
            }
            visitedEdge.add(outGoingEdge);
        }
    }
    log.error("No path found");
    return new ArrayList<>();
}

Example 4 with FlowEdge

use of org.apache.gobblin.runtime.api.FlowEdge in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method testWeightedGraphConstruction.

@Test
public void testWeightedGraphConstruction() {
    FlowSpec flowSpec = initFlowSpec();
    TopologySpec topologySpec = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR, TEST_SOURCE_NAME, TEST_HOP_NAME_A, TEST_HOP_NAME_B, TEST_SINK_NAME);
    this.compilerWithTemplateCalague.onAddSpec(topologySpec);
    // invocation of compileFlow trigger the weighedGraph construction
    this.compilerWithTemplateCalague.compileFlow(flowSpec);
    DirectedWeightedMultigraph<ServiceNode, FlowEdge> weightedGraph = compilerWithTemplateCalague.getWeightedGraph();
    Assert.assertTrue(weightedGraph.containsVertex(vertexSource));
    Assert.assertTrue(weightedGraph.containsVertex(vertexHopA));
    Assert.assertTrue(weightedGraph.containsVertex(vertexHopB));
    Assert.assertTrue(weightedGraph.containsVertex(vertexSink));
    FlowEdge edgeSrc2A = new LoadBasedFlowEdgeImpl(vertexSource, vertexHopA, topologySpec.getSpecExecutor());
    FlowEdge edgeA2B = new LoadBasedFlowEdgeImpl(vertexHopA, vertexHopB, topologySpec.getSpecExecutor());
    FlowEdge edgeB2Sink = new LoadBasedFlowEdgeImpl(vertexHopB, vertexSink, topologySpec.getSpecExecutor());
    Assert.assertTrue(weightedGraph.containsEdge(edgeSrc2A));
    Assert.assertTrue(weightedGraph.containsEdge(edgeA2B));
    Assert.assertTrue(weightedGraph.containsEdge(edgeB2Sink));
    Assert.assertTrue(edgeEqual(weightedGraph.getEdge(vertexSource, vertexHopA), edgeSrc2A));
    Assert.assertTrue(edgeEqual(weightedGraph.getEdge(vertexHopA, vertexHopB), edgeA2B));
    Assert.assertTrue(edgeEqual(weightedGraph.getEdge(vertexHopB, vertexSink), edgeB2Sink));
    this.compilerWithTemplateCalague.onDeleteSpec(topologySpec.getUri(), "");
}

Example 5 with FlowEdge

use of org.apache.gobblin.runtime.api.FlowEdge in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method populateTemplateMap.

// Use this function for
private void populateTemplateMap(WeightedMultigraph<ServiceNode, FlowEdge> weightedGraph, URI exempliedURI) {
    this.edgeTemplateMap.clear();
    Set<FlowEdge> allEdges = weightedGraph.edgeSet();
    for (FlowEdge edge : allEdges) {
        this.edgeTemplateMap.put(edge.getEdgeIdentity(), Arrays.asList(exempliedURI));
    }
}