Examples with CoGroupFunction org.apache.flink.api.common.functions.CoGroupFunction used on opensource projects

Example 1 with CoGroupFunction

use of org.apache.flink.api.common.functions.CoGroupFunction in project flink by apache.

the class BranchingPlansCompilerTest method testBranchingWithMultipleDataSinks.

/**
 * <pre>
 *
 *              (SINK A)
 *                  |    (SINK B)    (SINK C)
 *                CROSS    /          /
 *               /     \   |  +------+
 *              /       \  | /
 *          REDUCE      MATCH2
 *             |    +---/    \
 *              \  /          |
 *               MAP          |
 *                |           |
 *             COGROUP      MATCH1
 *             /     \     /     \
 *        (SRC A)    (SRC B)    (SRC C)
 * </pre>
 */
@Test
public void testBranchingWithMultipleDataSinks() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(DEFAULT_PARALLELISM);
        DataSet<Tuple2<Long, Long>> sourceA = env.generateSequence(1, 10000000).map(new Duplicator<Long>());
        DataSet<Tuple2<Long, Long>> sourceB = env.generateSequence(1, 10000000).map(new Duplicator<Long>());
        DataSet<Tuple2<Long, Long>> sourceC = env.generateSequence(1, 10000000).map(new Duplicator<Long>());
        DataSet<Tuple2<Long, Long>> mapped = sourceA.coGroup(sourceB).where(0).equalTo(1).with(new CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>>() {

            @Override
            public void coGroup(Iterable<Tuple2<Long, Long>> first, Iterable<Tuple2<Long, Long>> second, Collector<Tuple2<Long, Long>> out) {
            }
        }).map(new IdentityMapper<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined = sourceB.join(sourceC).where(0).equalTo(1).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined2 = mapped.join(joined).where(1).equalTo(1).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> reduced = mapped.groupBy(1).reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>());
        reduced.cross(joined2).output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
        joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
        joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
        Plan plan = env.createProgramPlan();
        OptimizedPlan oPlan = compileNoStats(plan);
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 2 with CoGroupFunction

use of org.apache.flink.api.common.functions.CoGroupFunction in project flink by apache.

the class DataSetAllroundTestProgram method main.

@SuppressWarnings("Convert2Lambda")
public static void main(String[] args) throws Exception {
    // get parameters
    ParameterTool params = ParameterTool.fromArgs(args);
    int loadFactor = Integer.parseInt(params.getRequired("loadFactor"));
    String outputPath = params.getRequired("outputPath");
    boolean infinite = params.getBoolean("infinite", false);
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    int numKeys = loadFactor * 128 * 1024;
    DataSet<Tuple2<String, Integer>> x1Keys;
    DataSet<Tuple2<String, Integer>> x2Keys = env.createInput(Generator.generate(numKeys * 32, 2)).setParallelism(4);
    DataSet<Tuple2<String, Integer>> x8Keys = env.createInput(Generator.generate(numKeys, 8)).setParallelism(4);
    if (infinite) {
        x1Keys = env.createInput(Generator.generateInfinitely(numKeys)).setParallelism(4);
    } else {
        x1Keys = env.createInput(Generator.generate(numKeys, 1)).setParallelism(4);
    }
    DataSet<Tuple2<String, Integer>> joined = x2Keys.map(x -> Tuple4.of("0-0", 0L, 1, x.f0)).returns(Types.TUPLE(Types.STRING, Types.LONG, Types.INT, Types.STRING)).join(x8Keys).where(3).equalTo(0).with((l, r) -> Tuple2.of(l.f3, 1)).returns(Types.TUPLE(Types.STRING, Types.INT)).groupBy(new KeySelector<Tuple2<String, Integer>, String>() {

        @Override
        public String getKey(Tuple2<String, Integer> value) {
            return value.f0;
        }
    }).reduce((value1, value2) -> Tuple2.of(value1.f0, value1.f1 + value2.f1));
    // co-group two datasets on their primary keys.
    // we filter both inputs such that only 6.25% of the keys overlap.
    // result: (key, cnt), #keys records with unique keys, cnt = (6.25%: 2, 93.75%: 1)
    DataSet<Tuple2<String, Integer>> coGrouped = x1Keys.filter(x -> x.f1 > 59).coGroup(x1Keys.filter(x -> x.f1 < 68)).where("f0").equalTo("f0").with((CoGroupFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>>) (l, r, out) -> {
        int cnt = 0;
        String key = "";
        for (Tuple2<String, Integer> t : l) {
            cnt++;
            key = t.f0;
        }
        for (Tuple2<String, Integer> t : r) {
            cnt++;
            key = t.f0;
        }
        out.collect(Tuple2.of(key, cnt));
    }).returns(Types.TUPLE(Types.STRING, Types.INT));
    // join datasets on keys (1-1 join) and replicate by 16 (previously computed count)
    // result: (key, cnt), 16 * #keys records, all keys preserved, cnt = (6.25%: 2, 93.75%: 1)
    DataSet<Tuple2<String, Integer>> joined2 = joined.join(coGrouped, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE).where(0).equalTo("f0").flatMap((FlatMapFunction<Tuple2<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>>) (p, out) -> {
        for (int i = 0; i < p.f0.f1; i++) {
            out.collect(Tuple2.of(p.f0.f0, p.f1.f1));
        }
    }).returns(Types.TUPLE(Types.STRING, Types.INT));
    // iteration. double the count field until all counts are at 32 or more
    // result: (key, cnt), 16 * #keys records, all keys preserved, cnt = (6.25%: 64, 93.75%: 32)
    IterativeDataSet<Tuple2<String, Integer>> initial = joined2.iterate(16);
    DataSet<Tuple2<String, Integer>> iteration = initial.map(x -> Tuple2.of(x.f0, x.f1 * 2)).returns(Types.TUPLE(Types.STRING, Types.INT));
    DataSet<Boolean> termination = iteration.flatMap((FlatMapFunction<Tuple2<String, Integer>, Boolean>) (x, out) -> {
        if (x.f1 < 32) {
            out.collect(false);
        }
    }).returns(Types.BOOLEAN);
    DataSet<Tuple2<Integer, Integer>> result = initial.closeWith(iteration, termination).groupBy(1).reduceGroup((GroupReduceFunction<Tuple2<String, Integer>, Tuple2<Integer, Integer>>) (g, out) -> {
        int key = 0;
        int cnt = 0;
        for (Tuple2<String, Integer> r : g) {
            key = r.f1;
            cnt++;
        }
        out.collect(Tuple2.of(key, cnt));
    }).returns(Types.TUPLE(Types.INT, Types.INT)).map(x -> Tuple2.of(x.f0, x.f1 / (loadFactor * 128))).returns(Types.TUPLE(Types.INT, Types.INT));
    // sort and emit result
    result.sortPartition(0, Order.ASCENDING).setParallelism(1).writeAsText(outputPath, FileSystem.WriteMode.OVERWRITE).setParallelism(1);
    env.execute();
}

Example 3 with CoGroupFunction

use of org.apache.flink.api.common.functions.CoGroupFunction in project flink by apache.

the class CoGroupRawOperator method translateToDataFlow.

@Override
protected org.apache.flink.api.common.operators.base.CoGroupRawOperatorBase<?, ?, OUT, ?> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
    String name = getName() != null ? getName() : "CoGroup at " + defaultName;
    try {
        keys1.areCompatible(keys2);
    } catch (IncompatibleKeysException e) {
        throw new InvalidProgramException("The types of the key fields do not match.", e);
    }
    if (keys1 instanceof Keys.ExpressionKeys && keys2 instanceof Keys.ExpressionKeys) {
        try {
            keys1.areCompatible(keys2);
        } catch (IncompatibleKeysException e) {
            throw new InvalidProgramException("The types of the key fields do not match.", e);
        }
        int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
        int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
        CoGroupRawOperatorBase<I1, I2, OUT, CoGroupFunction<I1, I2, OUT>> po = new CoGroupRawOperatorBase<>(function, new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()), logicalKeyPositions1, logicalKeyPositions2, name);
        // set inputs
        po.setFirstInput(input1);
        po.setSecondInput(input2);
        // set dop
        po.setParallelism(this.getParallelism());
        return po;
    } else {
        throw new UnsupportedOperationException("Unrecognized or incompatible key types.");
    }
}

Example 4 with CoGroupFunction

use of org.apache.flink.api.common.functions.CoGroupFunction in project flink by apache.

the class CoGroupSortTranslationTest method testGroupSortTuples.

@Test
public void testGroupSortTuples() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
        DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
        input1.coGroup(input2).where(1).equalTo(2).sortFirstGroup(0, Order.DESCENDING).sortSecondGroup(1, Order.ASCENDING).sortSecondGroup(0, Order.DESCENDING).with(new CoGroupFunction<Tuple2<Long, Long>, Tuple3<Long, Long, Long>, Long>() {

            @Override
            public void coGroup(Iterable<Tuple2<Long, Long>> first, Iterable<Tuple3<Long, Long, Long>> second, Collector<Long> out) {
            }
        }).output(new DiscardingOutputFormat<Long>());
        Plan p = env.createProgramPlan();
        GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();
        CoGroupOperatorBase<?, ?, ?, ?> coGroup = (CoGroupOperatorBase<?, ?, ?, ?>) sink.getInput();
        assertNotNull(coGroup.getGroupOrderForInputOne());
        assertNotNull(coGroup.getGroupOrderForInputTwo());
        assertEquals(1, coGroup.getGroupOrderForInputOne().getNumberOfFields());
        assertEquals(0, coGroup.getGroupOrderForInputOne().getFieldNumber(0).intValue());
        assertEquals(Order.DESCENDING, coGroup.getGroupOrderForInputOne().getOrder(0));
        assertEquals(2, coGroup.getGroupOrderForInputTwo().getNumberOfFields());
        assertEquals(1, coGroup.getGroupOrderForInputTwo().getFieldNumber(0).intValue());
        assertEquals(0, coGroup.getGroupOrderForInputTwo().getFieldNumber(1).intValue());
        assertEquals(Order.ASCENDING, coGroup.getGroupOrderForInputTwo().getOrder(0));
        assertEquals(Order.DESCENDING, coGroup.getGroupOrderForInputTwo().getOrder(1));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 5 with CoGroupFunction

use of org.apache.flink.api.common.functions.CoGroupFunction in project flink by apache.

the class CoGroupSortTranslationTest method testSortTuplesAndPojos.

@Test
public void testSortTuplesAndPojos() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
        DataSet<TestPoJo> input2 = env.fromElements(new TestPoJo());
        input1.coGroup(input2).where(1).equalTo("b").sortFirstGroup(0, Order.DESCENDING).sortSecondGroup("c", Order.ASCENDING).sortSecondGroup("a", Order.DESCENDING).with(new CoGroupFunction<Tuple2<Long, Long>, TestPoJo, Long>() {

            @Override
            public void coGroup(Iterable<Tuple2<Long, Long>> first, Iterable<TestPoJo> second, Collector<Long> out) {
            }
        }).output(new DiscardingOutputFormat<Long>());
        Plan p = env.createProgramPlan();
        GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();
        CoGroupOperatorBase<?, ?, ?, ?> coGroup = (CoGroupOperatorBase<?, ?, ?, ?>) sink.getInput();
        assertNotNull(coGroup.getGroupOrderForInputOne());
        assertNotNull(coGroup.getGroupOrderForInputTwo());
        assertEquals(1, coGroup.getGroupOrderForInputOne().getNumberOfFields());
        assertEquals(0, coGroup.getGroupOrderForInputOne().getFieldNumber(0).intValue());
        assertEquals(Order.DESCENDING, coGroup.getGroupOrderForInputOne().getOrder(0));
        assertEquals(2, coGroup.getGroupOrderForInputTwo().getNumberOfFields());
        assertEquals(2, coGroup.getGroupOrderForInputTwo().getFieldNumber(0).intValue());
        assertEquals(0, coGroup.getGroupOrderForInputTwo().getFieldNumber(1).intValue());
        assertEquals(Order.ASCENDING, coGroup.getGroupOrderForInputTwo().getOrder(0));
        assertEquals(Order.DESCENDING, coGroup.getGroupOrderForInputTwo().getOrder(1));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}