Examples with Tuple2ToVertexMap org.apache.flink.graph.utils.Tuple2ToVertexMap used on opensource projects

Example 1 with Tuple2ToVertexMap

use of org.apache.flink.graph.utils.Tuple2ToVertexMap in project flink by apache.

the class PregelCompilerTest method testPregelCompilerWithBroadcastVariable.

@SuppressWarnings("serial")
@Test
public void testPregelCompilerWithBroadcastVariable() {
    final String broadcastSetName = "broadcast";
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    // compose test program
    {
        DataSet<Long> bcVar = env.fromElements(1L);
        DataSet<Vertex<Long, Long>> initialVertices = env.fromElements(new Tuple2<>(1L, 1L), new Tuple2<>(2L, 2L)).map(new Tuple2ToVertexMap<>());
        DataSet<Edge<Long, NullValue>> edges = env.fromElements(new Tuple2<>(1L, 2L)).map(new MapFunction<Tuple2<Long, Long>, Edge<Long, NullValue>>() {

            public Edge<Long, NullValue> map(Tuple2<Long, Long> edge) {
                return new Edge<>(edge.f0, edge.f1, NullValue.getInstance());
            }
        });
        Graph<Long, Long, NullValue> graph = Graph.fromDataSet(initialVertices, edges, env);
        VertexCentricConfiguration parameters = new VertexCentricConfiguration();
        parameters.addBroadcastSet(broadcastSetName, bcVar);
        DataSet<Vertex<Long, Long>> result = graph.runVertexCentricIteration(new CCCompute(), null, 100, parameters).getVertices();
        result.output(new DiscardingOutputFormat<>());
    }
    Plan p = env.createProgramPlan("Pregel Connected Components");
    OptimizedPlan op = compileNoStats(p);
    // check the sink
    SinkPlanNode sink = op.getDataSinks().iterator().next();
    assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    assertEquals(DEFAULT_PARALLELISM, sink.getParallelism());
    // check the iteration
    WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
    assertEquals(DEFAULT_PARALLELISM, iteration.getParallelism());
    // check the solution set delta
    PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
    assertTrue(ssDelta instanceof SingleInputPlanNode);
    SingleInputPlanNode ssFlatMap = (SingleInputPlanNode) ((SingleInputPlanNode) (ssDelta)).getInput().getSource();
    assertEquals(DEFAULT_PARALLELISM, ssFlatMap.getParallelism());
    assertEquals(ShipStrategyType.FORWARD, ssFlatMap.getInput().getShipStrategy());
    // check the computation coGroup
    DualInputPlanNode computationCoGroup = (DualInputPlanNode) (ssFlatMap.getInput().getSource());
    assertEquals(DEFAULT_PARALLELISM, computationCoGroup.getParallelism());
    assertEquals(ShipStrategyType.FORWARD, computationCoGroup.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, computationCoGroup.getInput2().getShipStrategy());
    assertTrue(computationCoGroup.getInput2().getTempMode().isCached());
    assertEquals(new FieldList(0), computationCoGroup.getInput2().getShipStrategyKeys());
    // check that the initial partitioning is pushed out of the loop
    assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
    assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
}

Example 2 with Tuple2ToVertexMap

use of org.apache.flink.graph.utils.Tuple2ToVertexMap in project flink by apache.

the class PregelCompilerTest method testPregelCompiler.

@SuppressWarnings("serial")
@Test
public void testPregelCompiler() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    // compose test program
    {
        DataSet<Vertex<Long, Long>> initialVertices = env.fromElements(new Tuple2<>(1L, 1L), new Tuple2<>(2L, 2L)).map(new Tuple2ToVertexMap<>());
        DataSet<Edge<Long, NullValue>> edges = env.fromElements(new Tuple2<>(1L, 2L)).map(new MapFunction<Tuple2<Long, Long>, Edge<Long, NullValue>>() {

            public Edge<Long, NullValue> map(Tuple2<Long, Long> edge) {
                return new Edge<>(edge.f0, edge.f1, NullValue.getInstance());
            }
        });
        Graph<Long, Long, NullValue> graph = Graph.fromDataSet(initialVertices, edges, env);
        DataSet<Vertex<Long, Long>> result = graph.runVertexCentricIteration(new CCCompute(), null, 100).getVertices();
        result.output(new DiscardingOutputFormat<>());
    }
    Plan p = env.createProgramPlan("Pregel Connected Components");
    OptimizedPlan op = compileNoStats(p);
    // check the sink
    SinkPlanNode sink = op.getDataSinks().iterator().next();
    assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    assertEquals(DEFAULT_PARALLELISM, sink.getParallelism());
    // check the iteration
    WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
    assertEquals(DEFAULT_PARALLELISM, iteration.getParallelism());
    // check the solution set delta
    PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
    assertTrue(ssDelta instanceof SingleInputPlanNode);
    SingleInputPlanNode ssFlatMap = (SingleInputPlanNode) ((SingleInputPlanNode) (ssDelta)).getInput().getSource();
    assertEquals(DEFAULT_PARALLELISM, ssFlatMap.getParallelism());
    assertEquals(ShipStrategyType.FORWARD, ssFlatMap.getInput().getShipStrategy());
    // check the computation coGroup
    DualInputPlanNode computationCoGroup = (DualInputPlanNode) (ssFlatMap.getInput().getSource());
    assertEquals(DEFAULT_PARALLELISM, computationCoGroup.getParallelism());
    assertEquals(ShipStrategyType.FORWARD, computationCoGroup.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, computationCoGroup.getInput2().getShipStrategy());
    assertTrue(computationCoGroup.getInput2().getTempMode().isCached());
    assertEquals(new FieldList(0), computationCoGroup.getInput2().getShipStrategyKeys());
    // check that the initial partitioning is pushed out of the loop
    assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
    assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
}

Example 3 with Tuple2ToVertexMap

use of org.apache.flink.graph.utils.Tuple2ToVertexMap in project flink by apache.

the class PregelCompilerTest method testPregelWithCombiner.

@SuppressWarnings("serial")
@Test
public void testPregelWithCombiner() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    // compose test program
    {
        DataSet<Vertex<Long, Long>> initialVertices = env.fromElements(new Tuple2<>(1L, 1L), new Tuple2<>(2L, 2L)).map(new Tuple2ToVertexMap<>());
        DataSet<Edge<Long, NullValue>> edges = env.fromElements(new Tuple2<>(1L, 2L)).map(new MapFunction<Tuple2<Long, Long>, Edge<Long, NullValue>>() {

            public Edge<Long, NullValue> map(Tuple2<Long, Long> edge) {
                return new Edge<>(edge.f0, edge.f1, NullValue.getInstance());
            }
        });
        Graph<Long, Long, NullValue> graph = Graph.fromDataSet(initialVertices, edges, env);
        DataSet<Vertex<Long, Long>> result = graph.runVertexCentricIteration(new CCCompute(), new CCCombiner(), 100).getVertices();
        result.output(new DiscardingOutputFormat<>());
    }
    Plan p = env.createProgramPlan("Pregel Connected Components");
    OptimizedPlan op = compileNoStats(p);
    // check the sink
    SinkPlanNode sink = op.getDataSinks().iterator().next();
    assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    assertEquals(DEFAULT_PARALLELISM, sink.getParallelism());
    // check the iteration
    WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
    assertEquals(DEFAULT_PARALLELISM, iteration.getParallelism());
    // check the combiner
    SingleInputPlanNode combiner = (SingleInputPlanNode) iteration.getInput2().getSource();
    assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
    // check the solution set delta
    PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
    assertTrue(ssDelta instanceof SingleInputPlanNode);
    SingleInputPlanNode ssFlatMap = (SingleInputPlanNode) ((SingleInputPlanNode) (ssDelta)).getInput().getSource();
    assertEquals(DEFAULT_PARALLELISM, ssFlatMap.getParallelism());
    assertEquals(ShipStrategyType.FORWARD, ssFlatMap.getInput().getShipStrategy());
    // check the computation coGroup
    DualInputPlanNode computationCoGroup = (DualInputPlanNode) (ssFlatMap.getInput().getSource());
    assertEquals(DEFAULT_PARALLELISM, computationCoGroup.getParallelism());
    assertEquals(ShipStrategyType.FORWARD, computationCoGroup.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, computationCoGroup.getInput2().getShipStrategy());
    assertTrue(computationCoGroup.getInput2().getTempMode().isCached());
    assertEquals(new FieldList(0), computationCoGroup.getInput2().getShipStrategyKeys());
    // check that the initial partitioning is pushed out of the loop
    assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
    assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
}