Examples with Tuple2 org.apache.flink.api.java.tuple.Tuple2 used on opensource projects

Example 91 with Tuple2

use of org.apache.flink.api.java.tuple.Tuple2 in project flink by apache.

the class SumMinMaxITCase method testGroupedAggregate.

@Test
public void testGroupedAggregate() throws Exception {
    /*
		 * Grouped Aggregate
		 */
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
    DataSet<Tuple2<Long, Integer>> aggregateDs = ds.groupBy(1).sum(0).project(1, 0);
    List<Tuple2<Long, Integer>> result = aggregateDs.collect();
    String expected = "1,1\n" + "2,5\n" + "3,15\n" + "4,34\n" + "5,65\n" + "6,111\n";
    compareResultAsTuples(result, expected);
}

Example 92 with Tuple2

use of org.apache.flink.api.java.tuple.Tuple2 in project flink by apache.

the class SumMinMaxITCase method testSumMaxAndProject.

@Test
public void testSumMaxAndProject() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
    DataSet<Tuple2<Integer, Long>> sumDs = ds.sum(0).andMax(1).project(0, 1);
    List<Tuple2<Integer, Long>> result = sumDs.collect();
    String expected = "231,6\n";
    compareResultAsTuples(result, expected);
}

Example 93 with Tuple2

use of org.apache.flink.api.java.tuple.Tuple2 in project flink by apache.

the class CheckpointCoordinatorTest method serializeTogetherAndTrackOffsets.

public static Tuple2<byte[], List<long[]>> serializeTogetherAndTrackOffsets(List<List<? extends Serializable>> serializables) throws IOException {
    List<long[]> offsets = new ArrayList<>(serializables.size());
    List<byte[]> serializedGroupValues = new ArrayList<>();
    int runningGroupsOffset = 0;
    for (List<? extends Serializable> list : serializables) {
        long[] currentOffsets = new long[list.size()];
        offsets.add(currentOffsets);
        for (int i = 0; i < list.size(); ++i) {
            currentOffsets[i] = runningGroupsOffset;
            byte[] serializedValue = InstantiationUtil.serializeObject(list.get(i));
            serializedGroupValues.add(serializedValue);
            runningGroupsOffset += serializedValue.length;
        }
    }
    //write all generated values in a single byte array, which is index by groupOffsetsInFinalByteArray
    byte[] allSerializedValuesConcatenated = new byte[runningGroupsOffset];
    runningGroupsOffset = 0;
    for (byte[] serializedGroupValue : serializedGroupValues) {
        System.arraycopy(serializedGroupValue, 0, allSerializedValuesConcatenated, runningGroupsOffset, serializedGroupValue.length);
        runningGroupsOffset += serializedGroupValue.length;
    }
    return new Tuple2<>(allSerializedValuesConcatenated, offsets);
}

Example 94 with Tuple2

use of org.apache.flink.api.java.tuple.Tuple2 in project flink by apache.

the class JoinCancelingITCase method executeTaskWithGenerator.

private void executeTaskWithGenerator(JoinFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> joiner, int keys, int vals, int msecsTillCanceling, int maxTimeTillCanceled) throws Exception {
    UniformIntTupleGenerator g = new UniformIntTupleGenerator(keys, vals, false);
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple2<Integer, Integer>> input1 = env.createInput(new UniformIntTupleGeneratorInputFormat(keys, vals));
    DataSet<Tuple2<Integer, Integer>> input2 = env.createInput(new UniformIntTupleGeneratorInputFormat(keys, vals));
    input1.join(input2, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE).where(0).equalTo(0).with(joiner).output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());
    env.setParallelism(parallelism);
    runAndCancelJob(env.createProgramPlan(), msecsTillCanceling, maxTimeTillCanceled);
}

Example 95 with Tuple2

use of org.apache.flink.api.java.tuple.Tuple2 in project flink by apache.

the class AbstractEventTimeWindowCheckpointingITCase method testPreAggregatedTumblingTimeWindow.

@Test
public void testPreAggregatedTumblingTimeWindow() {
    final int NUM_ELEMENTS_PER_KEY = numElementsPerKey();
    final int WINDOW_SIZE = windowSize();
    final int NUM_KEYS = numKeys();
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0));
        env.getConfig().disableSysoutLogging();
        env.setStateBackend(this.stateBackend);
        env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)).rebalance().keyBy(0).timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS)).reduce(new ReduceFunction<Tuple2<Long, IntType>>() {

            @Override
            public Tuple2<Long, IntType> reduce(Tuple2<Long, IntType> a, Tuple2<Long, IntType> b) {
                return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
            }
        }, new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> input, Collector<Tuple4<Long, Long, Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                for (Tuple2<Long, IntType> in : input) {
                    out.collect(new Tuple4<>(in.f0, window.getStart(), window.getEnd(), in.f1));
                }
            }
        }).addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE)).setParallelism(1);
        tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}