Examples with ScatterGatherConfiguration org.apache.flink.graph.spargel.ScatterGatherConfiguration used on opensource projects

Example 6 with ScatterGatherConfiguration

use of org.apache.flink.graph.spargel.ScatterGatherConfiguration in project flink by apache.

the class ScatterGatherConfigurationITCase method testRunWithConfiguration.

@Test
public void testRunWithConfiguration() throws Exception {
    /*
		 * Test Graph's runScatterGatherIteration when configuration parameters are provided
		 */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    Graph<Long, Long, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongVertices(), TestGraphUtils.getLongLongEdges(), env).mapVertices(new AssignOneMapper());
    // create the configuration object
    ScatterGatherConfiguration parameters = new ScatterGatherConfiguration();
    parameters.addBroadcastSetForScatterFunction("messagingBcastSet", env.fromElements(4, 5, 6));
    parameters.addBroadcastSetForGatherFunction("updateBcastSet", env.fromElements(1, 2, 3));
    parameters.registerAggregator("superstepAggregator", new LongSumAggregator());
    parameters.setOptNumVertices(true);
    Graph<Long, Long, Long> res = graph.runScatterGatherIteration(new MessageFunction(), new UpdateFunction(), 10, parameters);
    DataSet<Vertex<Long, Long>> data = res.getVertices();
    List<Vertex<Long, Long>> result = data.collect();
    expectedResult = "1,11\n" + "2,11\n" + "3,11\n" + "4,11\n" + "5,11";
    compareResultAsTuples(result, expectedResult);
}

Example 7 with ScatterGatherConfiguration

use of org.apache.flink.graph.spargel.ScatterGatherConfiguration in project flink by apache.

the class ScatterGatherConfigurationITCase method testSendToAllDirectionALL.

@Test
public void testSendToAllDirectionALL() throws Exception {
    /*
		 * Test that sendMessageToAllNeighbors() works correctly
		 * when the direction is set to ALL
		 */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    Graph<Long, HashSet<Long>, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongVertices(), TestGraphUtils.getLongLongEdges(), env).mapVertices(new InitialiseHashSetMapper());
    // configure the iteration
    ScatterGatherConfiguration parameters = new ScatterGatherConfiguration();
    parameters.setDirection(EdgeDirection.ALL);
    DataSet<Vertex<Long, HashSet<Long>>> resultedVertices = graph.runScatterGatherIteration(new SendMsgToAll(), new VertexUpdateDirection(), 5, parameters).getVertices();
    List<Vertex<Long, HashSet<Long>>> result = resultedVertices.collect();
    expectedResult = "1,[2, 3, 5]\n" + "2,[1, 3]\n" + "3,[1, 2, 4, 5]\n" + "4,[3, 5]\n" + "5,[1, 3, 4]";
    compareResultAsTuples(result, expectedResult);
}

Example 8 with ScatterGatherConfiguration

use of org.apache.flink.graph.spargel.ScatterGatherConfiguration in project flink by apache.

the class IncrementalSSSP method main.

public static void main(String[] args) throws Exception {
    if (!parseParameters(args)) {
        return;
    }
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    Edge<Long, Double> edgeToBeRemoved = getEdgeToBeRemoved();
    Graph<Long, Double, Double> graph = IncrementalSSSP.getGraph(env);
    // Assumption: all minimum weight paths are kept
    Graph<Long, Double, Double> ssspGraph = IncrementalSSSP.getSSSPGraph(env);
    // remove the edge
    graph.removeEdge(edgeToBeRemoved);
    // configure the iteration
    ScatterGatherConfiguration parameters = new ScatterGatherConfiguration();
    if (isInSSSP(edgeToBeRemoved, ssspGraph.getEdges())) {
        parameters.setDirection(EdgeDirection.IN);
        parameters.setOptDegrees(true);
        // run the scatter-gather iteration to propagate info
        Graph<Long, Double, Double> result = ssspGraph.runScatterGatherIteration(new InvalidateMessenger(edgeToBeRemoved), new VertexDistanceUpdater(), maxIterations, parameters);
        DataSet<Vertex<Long, Double>> resultedVertices = result.getVertices();
        // Emit results
        if (fileOutput) {
            resultedVertices.writeAsCsv(outputPath, "\n", ",");
            env.execute("Incremental SSSP Example");
        } else {
            resultedVertices.print();
        }
    } else {
        // print the vertices
        if (fileOutput) {
            graph.getVertices().writeAsCsv(outputPath, "\n", ",");
            env.execute("Incremental SSSP Example");
        } else {
            graph.getVertices().print();
        }
    }
}

Example 9 with ScatterGatherConfiguration

use of org.apache.flink.graph.spargel.ScatterGatherConfiguration in project flink by apache.

the class IncrementalSSSPITCase method testIncrementalSSSPNonSPEdge.

@Test
public void testIncrementalSSSPNonSPEdge() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Vertex<Long, Double>> vertices = IncrementalSSSPData.getDefaultVertexDataSet(env);
    DataSet<Edge<Long, Double>> edges = IncrementalSSSPData.getDefaultEdgeDataSet(env);
    DataSet<Edge<Long, Double>> edgesInSSSP = IncrementalSSSPData.getDefaultEdgesInSSSP(env);
    // the edge to be removed is a non-SP edge
    Edge<Long, Double> edgeToBeRemoved = new Edge<>(3L, 5L, 5.0);
    Graph<Long, Double, Double> graph = Graph.fromDataSet(vertices, edges, env);
    // Assumption: all minimum weight paths are kept
    Graph<Long, Double, Double> ssspGraph = Graph.fromDataSet(vertices, edgesInSSSP, env);
    // remove the edge
    graph.removeEdge(edgeToBeRemoved);
    // configure the iteration
    ScatterGatherConfiguration parameters = new ScatterGatherConfiguration();
    if (IncrementalSSSP.isInSSSP(edgeToBeRemoved, edgesInSSSP)) {
        parameters.setDirection(EdgeDirection.IN);
        parameters.setOptDegrees(true);
        // run the scatter gather iteration to propagate info
        Graph<Long, Double, Double> result = ssspGraph.runScatterGatherIteration(new IncrementalSSSP.InvalidateMessenger(edgeToBeRemoved), new IncrementalSSSP.VertexDistanceUpdater(), IncrementalSSSPData.NUM_VERTICES, parameters);
        DataSet<Vertex<Long, Double>> resultedVertices = result.getVertices();
        resultedVertices.writeAsCsv(resultPath, "\n", ",");
        env.execute();
    } else {
        vertices.writeAsCsv(resultPath, "\n", ",");
        env.execute();
    }
    expected = IncrementalSSSPData.VERTICES;
}

Example 10 with ScatterGatherConfiguration

use of org.apache.flink.graph.spargel.ScatterGatherConfiguration in project flink by apache.

the class ScatterGatherConfigurationITCase method testIterationINDirection.

@Test
public void testIterationINDirection() throws Exception {
    /*
		 * Test that if the direction parameter is set to IN,
		 * messages are collected from the out-neighbors and sent to the in-neighbors.
		 */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    Graph<Long, HashSet<Long>, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongVertices(), TestGraphUtils.getLongLongEdges(), env).mapVertices(new InitialiseHashSetMapper());
    // configure the iteration
    ScatterGatherConfiguration parameters = new ScatterGatherConfiguration();
    parameters.setDirection(EdgeDirection.IN);
    DataSet<Vertex<Long, HashSet<Long>>> resultedVertices = graph.runScatterGatherIteration(new IdMessengerSrc(), new VertexUpdateDirection(), 5, parameters).getVertices();
    List<Vertex<Long, HashSet<Long>>> result = resultedVertices.collect();
    expectedResult = "1,[2, 3]\n" + "2,[3]\n" + "3,[4, 5]\n" + "4,[5]\n" + "5,[1]";
    compareResultAsTuples(result, expectedResult);
}