Examples with Tuple4 org.apache.flink.api.java.tuple.Tuple4 used on opensource projects

Example 36 with Tuple4

use of org.apache.flink.api.java.tuple.Tuple4 in project flink by apache.

the class MigrationV0ToV1Test method createTaskStatesOld.

private static Collection<org.apache.flink.migration.runtime.checkpoint.TaskState> createTaskStatesOld(int numTaskStates, int numSubtaskStates) throws Exception {
    List<org.apache.flink.migration.runtime.checkpoint.TaskState> taskStates = new ArrayList<>(numTaskStates);
    for (int i = 0; i < numTaskStates; i++) {
        org.apache.flink.migration.runtime.checkpoint.TaskState taskState = new org.apache.flink.migration.runtime.checkpoint.TaskState(new JobVertexID(), numSubtaskStates);
        for (int j = 0; j < numSubtaskStates; j++) {
            StreamTaskState[] streamTaskStates = new StreamTaskState[2];
            for (int k = 0; k < streamTaskStates.length; k++) {
                StreamTaskState state = new StreamTaskState();
                Tuple4<Integer, Integer, Integer, Integer> testState = new Tuple4<>(0, i, j, k);
                if (j % 4 != 0) {
                    state.setFunctionState(new SerializedStateHandle<Serializable>(testState));
                }
                testState = new Tuple4<>(1, i, j, k);
                state.setOperatorState(new SerializedStateHandle<>(testState));
                if ((0 == k) && (i % 3 != 0)) {
                    HashMap<String, KvStateSnapshot<?, ?, ?, ?>> testKeyedState = new HashMap<>(2);
                    for (int l = 0; l < 2; ++l) {
                        String name = "keyed-" + l;
                        KvStateSnapshot<?, ?, ?, ?> testKeyedSnapshot = new MemValueState.Snapshot<>(IntSerializer.INSTANCE, VoidNamespaceSerializer.INSTANCE, IntSerializer.INSTANCE, new ValueStateDescriptor<>(name, Integer.class, 0), new byte[] { (byte) i, (byte) j });
                        testKeyedState.put(name, testKeyedSnapshot);
                    }
                    state.setKvStates(testKeyedState);
                }
                streamTaskStates[k] = state;
            }
            StreamTaskStateList streamTaskStateList = new StreamTaskStateList(streamTaskStates);
            org.apache.flink.migration.util.SerializedValue<org.apache.flink.migration.runtime.state.StateHandle<?>> handle = new org.apache.flink.migration.util.SerializedValue<org.apache.flink.migration.runtime.state.StateHandle<?>>(streamTaskStateList);
            taskState.putState(j, new org.apache.flink.migration.runtime.checkpoint.SubtaskState(handle, 0, 0));
        }
        taskStates.add(taskState);
    }
    return taskStates;
}

Example 37 with Tuple4

use of org.apache.flink.api.java.tuple.Tuple4 in project flink by apache.

the class EventTimeAllWindowCheckpointingITCase method testTumblingTimeWindow.

// ------------------------------------------------------------------------
@Test
public void testTumblingTimeWindow() {
    final int NUM_ELEMENTS_PER_KEY = 3000;
    final int WINDOW_SIZE = 100;
    final int NUM_KEYS = 1;
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0));
        env.getConfig().disableSysoutLogging();
        env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)).rebalance().timeWindowAll(Time.of(WINDOW_SIZE, MILLISECONDS)).apply(new RichAllWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(TimeWindow window, Iterable<Tuple2<Long, IntType>> values, Collector<Tuple4<Long, Long, Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                int sum = 0;
                long key = -1;
                for (Tuple2<Long, IntType> value : values) {
                    sum += value.f1.value;
                    key = value.f0;
                }
                out.collect(new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum)));
            }
        }).addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE)).setParallelism(1);
        tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 38 with Tuple4

use of org.apache.flink.api.java.tuple.Tuple4 in project flink by apache.

the class EventTimeAllWindowCheckpointingITCase method testPreAggregatedSlidingTimeWindow.

@Test
public void testPreAggregatedSlidingTimeWindow() {
    final int NUM_ELEMENTS_PER_KEY = 3000;
    final int WINDOW_SIZE = 1000;
    final int WINDOW_SLIDE = 100;
    final int NUM_KEYS = 1;
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0));
        env.getConfig().disableSysoutLogging();
        env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)).rebalance().timeWindowAll(Time.of(WINDOW_SIZE, MILLISECONDS), Time.of(WINDOW_SLIDE, MILLISECONDS)).reduce(new ReduceFunction<Tuple2<Long, IntType>>() {

            @Override
            public Tuple2<Long, IntType> reduce(Tuple2<Long, IntType> a, Tuple2<Long, IntType> b) {
                return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
            }
        }, new RichAllWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(TimeWindow window, Iterable<Tuple2<Long, IntType>> input, Collector<Tuple4<Long, Long, Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                for (Tuple2<Long, IntType> in : input) {
                    out.collect(new Tuple4<>(in.f0, window.getStart(), window.getEnd(), in.f1));
                }
            }
        }).addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SLIDE)).setParallelism(1);
        tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 39 with Tuple4

use of org.apache.flink.api.java.tuple.Tuple4 in project flink by apache.

the class EventTimeAllWindowCheckpointingITCase method testPreAggregatedTumblingTimeWindow.

@Test
public void testPreAggregatedTumblingTimeWindow() {
    final int NUM_ELEMENTS_PER_KEY = 3000;
    final int WINDOW_SIZE = 100;
    final int NUM_KEYS = 1;
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0));
        env.getConfig().disableSysoutLogging();
        env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)).rebalance().timeWindowAll(Time.of(WINDOW_SIZE, MILLISECONDS)).reduce(new ReduceFunction<Tuple2<Long, IntType>>() {

            @Override
            public Tuple2<Long, IntType> reduce(Tuple2<Long, IntType> a, Tuple2<Long, IntType> b) {
                return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
            }
        }, new RichAllWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(TimeWindow window, Iterable<Tuple2<Long, IntType>> input, Collector<Tuple4<Long, Long, Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                for (Tuple2<Long, IntType> in : input) {
                    out.collect(new Tuple4<>(in.f0, window.getStart(), window.getEnd(), in.f1));
                }
            }
        }).addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE)).setParallelism(1);
        tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 40 with Tuple4

use of org.apache.flink.api.java.tuple.Tuple4 in project flink by apache.

the class EventTimeAllWindowCheckpointingITCase method testPreAggregatedFoldingTumblingTimeWindow.

@Test
public void testPreAggregatedFoldingTumblingTimeWindow() {
    final int NUM_ELEMENTS_PER_KEY = 3000;
    final int WINDOW_SIZE = 100;
    final int NUM_KEYS = 1;
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0));
        env.getConfig().disableSysoutLogging();
        env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)).rebalance().timeWindowAll(Time.of(WINDOW_SIZE, MILLISECONDS)).fold(new Tuple4<>(0L, 0L, 0L, new IntType(0)), new FoldFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>>() {

            @Override
            public Tuple4<Long, Long, Long, IntType> fold(Tuple4<Long, Long, Long, IntType> accumulator, Tuple2<Long, IntType> value) throws Exception {
                accumulator.f0 = value.f0;
                accumulator.f3 = new IntType(accumulator.f3.value + value.f1.value);
                return accumulator;
            }
        }, new RichAllWindowFunction<Tuple4<Long, Long, Long, IntType>, Tuple4<Long, Long, Long, IntType>, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(TimeWindow window, Iterable<Tuple4<Long, Long, Long, IntType>> input, Collector<Tuple4<Long, Long, Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                for (Tuple4<Long, Long, Long, IntType> in : input) {
                    out.collect(new Tuple4<>(in.f0, window.getStart(), window.getEnd(), in.f3));
                }
            }
        }).addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE)).setParallelism(1);
        tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}