Examples with ReduceFunction org.apache.flink.api.common.functions.ReduceFunction used on opensource projects

Example 21 with ReduceFunction

use of org.apache.flink.api.common.functions.ReduceFunction in project flink by apache.

the class StateDescriptorPassingTest method testReduceWindowAllState.

@Test
public void testReduceWindowAllState() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);
    // simulate ingestion time
    DataStream<File> src = env.fromElements(new File("/")).assignTimestampsAndWatermarks(WatermarkStrategy.<File>forMonotonousTimestamps().withTimestampAssigner((file, ts) -> System.currentTimeMillis()));
    SingleOutputStreamOperator<?> result = src.windowAll(TumblingEventTimeWindows.of(Time.milliseconds(1000))).reduce(new ReduceFunction<File>() {

        @Override
        public File reduce(File value1, File value2) {
            return null;
        }
    });
    validateStateDescriptorConfigured(result);
}

Example 22 with ReduceFunction

use of org.apache.flink.api.common.functions.ReduceFunction in project flink by apache.

the class StreamingRuntimeContextTest method testReducingStateInstantiation.

@Test
public void testReducingStateInstantiation() throws Exception {
    final ExecutionConfig config = new ExecutionConfig();
    config.registerKryoType(Path.class);
    final AtomicReference<Object> descriptorCapture = new AtomicReference<>();
    StreamingRuntimeContext context = createRuntimeContext(descriptorCapture, config);
    @SuppressWarnings("unchecked") ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class);
    ReducingStateDescriptor<TaskInfo> descr = new ReducingStateDescriptor<>("name", reducer, TaskInfo.class);
    context.getReducingState(descr);
    StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
    TypeSerializer<?> serializer = descrIntercepted.getSerializer();
    // check that the Path class is really registered, i.e., the execution config was applied
    assertTrue(serializer instanceof KryoSerializer);
    assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}

Example 23 with ReduceFunction

use of org.apache.flink.api.common.functions.ReduceFunction in project flink by apache.

the class ReinterpretDataStreamAsKeyedStreamITCase method testReinterpretAsKeyedStream.

/**
 * This test checks that reinterpreting a data stream to a keyed stream works as expected. This
 * test consists of two jobs. The first job materializes a keyBy into files, one files per
 * partition. The second job opens the files created by the first jobs as sources (doing the
 * correct assignment of files to partitions) and reinterprets the sources as keyed, because we
 * know they have been partitioned in a keyBy from the first job.
 */
@Test
public void testReinterpretAsKeyedStream() throws Exception {
    final int maxParallelism = 8;
    final int numEventsPerInstance = 100;
    final int parallelism = 3;
    final int numTotalEvents = numEventsPerInstance * parallelism;
    final int numUniqueKeys = 100;
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setMaxParallelism(maxParallelism);
    env.setParallelism(parallelism);
    env.enableCheckpointing(100);
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0L));
    final List<File> partitionFiles = new ArrayList<>(parallelism);
    for (int i = 0; i < parallelism; ++i) {
        File partitionFile = temporaryFolder.newFile();
        partitionFiles.add(i, partitionFile);
    }
    env.addSource(new RandomTupleSource(numEventsPerInstance, numUniqueKeys)).keyBy(0).addSink(new ToPartitionFileSink(partitionFiles));
    env.execute();
    DataStream<Tuple2<Integer, Integer>> source = env.addSource(new FromPartitionFileSource(partitionFiles)).assignTimestampsAndWatermarks(IngestionTimeWatermarkStrategy.create());
    DataStreamUtils.reinterpretAsKeyedStream(source, (KeySelector<Tuple2<Integer, Integer>, Integer>) value -> value.f0, TypeInformation.of(Integer.class)).window(TumblingEventTimeWindows.of(Time.seconds(// test that also timers and aggregated state work as
    1))).reduce((ReduceFunction<Tuple2<Integer, Integer>>) (value1, value2) -> new Tuple2<>(value1.f0, value1.f1 + value2.f1)).addSink(new ValidatingSink(numTotalEvents)).setParallelism(1);
    env.execute();
}

Example 24 with ReduceFunction

use of org.apache.flink.api.common.functions.ReduceFunction in project flink by apache.

the class IterationWithAllReducerITCase method testProgram.

@Override
protected void testProgram() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(4);
    DataSet<String> initialInput = env.fromElements("1", "1", "1", "1", "1", "1", "1", "1");
    IterativeDataSet<String> iteration = initialInput.iterate(5).name("Loop");
    DataSet<String> sumReduce = iteration.reduce(new ReduceFunction<String>() {

        @Override
        public String reduce(String value1, String value2) throws Exception {
            return value1;
        }
    }).name("Compute sum (Reduce)");
    List<String> result = iteration.closeWith(sumReduce).collect();
    compareResultAsText(result, EXPECTED);
}

Example 25 with ReduceFunction

use of org.apache.flink.api.common.functions.ReduceFunction in project flink by apache.

the class WordCountSubclassPOJOITCase method testProgram.

@Override
protected void testProgram() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<String> text = env.readTextFile(textPath);
    DataSet<WCBase> counts = text.flatMap(new Tokenizer()).groupBy("word").reduce(new ReduceFunction<WCBase>() {

        private static final long serialVersionUID = 1L;

        public WCBase reduce(WCBase value1, WCBase value2) {
            WC wc1 = (WC) value1;
            WC wc2 = (WC) value2;
            return new WC(value1.word, wc1.secretCount + wc2.secretCount);
        }
    }).map(new MapFunction<WCBase, WCBase>() {

        @Override
        public WCBase map(WCBase value) throws Exception {
            WC wc = (WC) value;
            wc.count = wc.secretCount;
            return wc;
        }
    });
    counts.writeAsText(resultPath);
    env.execute("WordCount with custom data types example");
}