Examples with Edge org.apache.flink.graph.Edge used on opensource projects

Example 6 with Edge

use of org.apache.flink.graph.Edge in project flink by apache.

the class GSATranslationTest method testTranslation.

@Test
public void testTranslation() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Long> bcGather = env.fromElements(1L);
    DataSet<Long> bcSum = env.fromElements(1L);
    DataSet<Long> bcApply = env.fromElements(1L);
    DataSet<Vertex<Long, Long>> result;
    // ------------ construct the test program ------------------
    DataSet<Edge<Long, NullValue>> edges = env.fromElements(new Tuple3<>(1L, 2L, NullValue.getInstance())).map(new Tuple3ToEdgeMap<>());
    Graph<Long, Long, NullValue> graph = Graph.fromDataSet(edges, new InitVertices(), env);
    GSAConfiguration parameters = new GSAConfiguration();
    parameters.registerAggregator(AGGREGATOR_NAME, new LongSumAggregator());
    parameters.setName(ITERATION_NAME);
    parameters.setParallelism(ITERATION_parallelism);
    parameters.addBroadcastSetForGatherFunction(BC_SET_GATHER_NAME, bcGather);
    parameters.addBroadcastSetForSumFunction(BC_SET_SUM_NAME, bcSum);
    parameters.addBroadcastSetForApplyFunction(BC_SET_APLLY_NAME, bcApply);
    result = graph.runGatherSumApplyIteration(new GatherNeighborIds(), new SelectMinId(), new UpdateComponentId(), NUM_ITERATIONS, parameters).getVertices();
    result.output(new DiscardingOutputFormat<>());
    // ------------- validate the java program ----------------
    assertTrue(result instanceof DeltaIterationResultSet);
    DeltaIterationResultSet<?, ?> resultSet = (DeltaIterationResultSet<?, ?>) result;
    DeltaIteration<?, ?> iteration = resultSet.getIterationHead();
    // check the basic iteration properties
    assertEquals(NUM_ITERATIONS, resultSet.getMaxIterations());
    assertArrayEquals(new int[] { 0 }, resultSet.getKeyPositions());
    assertEquals(ITERATION_parallelism, iteration.getParallelism());
    assertEquals(ITERATION_NAME, iteration.getName());
    assertEquals(AGGREGATOR_NAME, iteration.getAggregators().getAllRegisteredAggregators().iterator().next().getName());
    // validate that the semantic properties are set as they should
    TwoInputUdfOperator<?, ?, ?, ?> solutionSetJoin = (TwoInputUdfOperator<?, ?, ?, ?>) resultSet.getNextWorkset();
    assertTrue(solutionSetJoin.getSemanticProperties().getForwardingTargetFields(0, 0).contains(0));
    assertTrue(solutionSetJoin.getSemanticProperties().getForwardingTargetFields(1, 0).contains(0));
    SingleInputUdfOperator<?, ?, ?> sumReduce = (SingleInputUdfOperator<?, ?, ?>) solutionSetJoin.getInput1();
    SingleInputUdfOperator<?, ?, ?> gatherMap = (SingleInputUdfOperator<?, ?, ?>) sumReduce.getInput();
    // validate that the broadcast sets are forwarded
    assertEquals(bcGather, gatherMap.getBroadcastSets().get(BC_SET_GATHER_NAME));
    assertEquals(bcSum, sumReduce.getBroadcastSets().get(BC_SET_SUM_NAME));
    assertEquals(bcApply, solutionSetJoin.getBroadcastSets().get(BC_SET_APLLY_NAME));
}

Example 7 with Edge

use of org.apache.flink.graph.Edge in project flink by apache.

the class GSACompilerTest method testGSACompiler.

@Test
public void testGSACompiler() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    // compose test program
    DataSet<Edge<Long, NullValue>> edges = env.fromElements(new Tuple3<>(1L, 2L, NullValue.getInstance())).map(new Tuple3ToEdgeMap<>());
    Graph<Long, Long, NullValue> graph = Graph.fromDataSet(edges, new InitVertices(), env);
    DataSet<Vertex<Long, Long>> result = graph.runGatherSumApplyIteration(new GatherNeighborIds(), new SelectMinId(), new UpdateComponentId(), 100).getVertices();
    result.output(new DiscardingOutputFormat<>());
    Plan p = env.createProgramPlan("GSA Connected Components");
    OptimizedPlan op = compileNoStats(p);
    // check the sink
    SinkPlanNode sink = op.getDataSinks().iterator().next();
    assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    assertEquals(DEFAULT_PARALLELISM, sink.getParallelism());
    assertEquals(PartitioningProperty.HASH_PARTITIONED, sink.getGlobalProperties().getPartitioning());
    // check the iteration
    WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
    assertEquals(DEFAULT_PARALLELISM, iteration.getParallelism());
    // check the solution set join and the delta
    PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
    assertTrue(ssDelta instanceof // this is only true if the update function preserves
    DualInputPlanNode);
    // the partitioning
    DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
    assertEquals(DEFAULT_PARALLELISM, ssJoin.getParallelism());
    assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
    assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
    // check the workset set join
    SingleInputPlanNode sumReducer = (SingleInputPlanNode) ssJoin.getInput1().getSource();
    SingleInputPlanNode gatherMapper = (SingleInputPlanNode) sumReducer.getInput().getSource();
    DualInputPlanNode edgeJoin = (DualInputPlanNode) gatherMapper.getInput().getSource();
    assertEquals(DEFAULT_PARALLELISM, edgeJoin.getParallelism());
    // input1 is the workset
    assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput1().getShipStrategy());
    // input2 is the edges
    assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput2().getShipStrategy());
    assertTrue(edgeJoin.getInput2().getTempMode().isCached());
    assertEquals(new FieldList(0), edgeJoin.getInput2().getShipStrategyKeys());
}

Example 8 with Edge

use of org.apache.flink.graph.Edge in project flink by apache.

the class CommunityDetectionTest method testWithRMatGraph.

@Test
public void testWithRMatGraph() throws Exception {
    Graph<LongValue, Long, Double> result = undirectedRMatGraph(8, 4).mapVertices(v -> v.getId().getValue(), new TypeHint<Vertex<LongValue, Long>>() {
    }.getTypeInfo()).mapEdges(e -> (double) e.getTarget().getValue() - e.getSource().getValue(), new TypeHint<Edge<LongValue, Double>>() {
    }.getTypeInfo()).run(new CommunityDetection<>(10, 0.5));
    Checksum checksum = new ChecksumHashCode<Vertex<LongValue, Long>>().run(result.getVertices()).execute();
    assertEquals(184, checksum.getCount());
    assertEquals(0x00000000000cdc96L, checksum.getChecksum());
}

Example 9 with Edge

use of org.apache.flink.graph.Edge in project flink by apache.

the class EdgeSourceDegreeTest method testWithRMatGraph.

@Test
public void testWithRMatGraph() throws Exception {
    DataSet<Edge<LongValue, Tuple2<NullValue, LongValue>>> sourceDegreeOnSourceId = undirectedRMatGraph(10, 16).run(new EdgeSourceDegree<LongValue, NullValue, NullValue>().setReduceOnTargetId(false));
    Checksum checksumOnSourceId = new ChecksumHashCode<Edge<LongValue, Tuple2<NullValue, LongValue>>>().run(sourceDegreeOnSourceId).execute();
    assertEquals(20884, checksumOnSourceId.getCount());
    assertEquals(0x000000019d8f0070L, checksumOnSourceId.getChecksum());
    DataSet<Edge<LongValue, Tuple2<NullValue, LongValue>>> sourceDegreeOnTargetId = undirectedRMatGraph(10, 16).run(new EdgeSourceDegree<LongValue, NullValue, NullValue>().setReduceOnTargetId(true));
    Checksum checksumOnTargetId = new ChecksumHashCode<Edge<LongValue, Tuple2<NullValue, LongValue>>>().run(sourceDegreeOnTargetId).execute();
    assertEquals(checksumOnTargetId, checksumOnTargetId);
}

Example 10 with Edge

use of org.apache.flink.graph.Edge in project flink by apache.

the class TranslateTest method setup.

@Before
public void setup() {
    ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();
    int count = 10;
    List<Vertex<LongValue, LongValue>> vertexList = new LinkedList<>();
    List<Edge<LongValue, LongValue>> edgeList = new LinkedList<>();
    for (long l = 0; l < count; l++) {
        LongValue lv0 = new LongValue(l);
        LongValue lv1 = new LongValue(l + 1);
        LongValue lv2 = new LongValue(l + 2);
        vertexList.add(new Vertex<>(lv0, lv1));
        edgeList.add(new Edge<>(lv0, lv1, lv2));
    }
    graph = Graph.fromCollection(vertexList, edgeList, env);
}