Examples with ReduceFunction org.apache.flink.api.common.functions.ReduceFunction used on opensource projects

Example 11 with ReduceFunction

use of org.apache.flink.api.common.functions.ReduceFunction in project flink by apache.

the class ReduceCombineDriverTest method testImmutableEmpty.

@Test
public void testImmutableEmpty() {
    try {
        TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context = new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>(1024 * 1024);
        context.getTaskConfig().setRelativeMemoryDriver(0.5);
        List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
        Collections.shuffle(data);
        TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
        MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
        context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
        TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
        GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String, Integer>>(typeInfo.createSerializer(new ExecutionConfig()));
        context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
        context.setComparator1(comparator);
        context.setCollector(result);
        ReduceCombineDriver<Tuple2<String, Integer>> driver = new ReduceCombineDriver<Tuple2<String, Integer>>();
        driver.setup(context);
        driver.prepare();
        driver.run();
        Assert.assertEquals(0, result.getList().size());
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 12 with ReduceFunction

use of org.apache.flink.api.common.functions.ReduceFunction in project flink by apache.

the class WordCountSubclassPOJOITCase method testProgram.

@Override
protected void testProgram() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<String> text = env.readTextFile(textPath);
    DataSet<WCBase> counts = text.flatMap(new Tokenizer()).groupBy("word").reduce(new ReduceFunction<WCBase>() {

        private static final long serialVersionUID = 1L;

        public WCBase reduce(WCBase value1, WCBase value2) {
            WC wc1 = (WC) value1;
            WC wc2 = (WC) value2;
            return new WC(value1.word, wc1.secretCount + wc2.secretCount);
        }
    }).map(new MapFunction<WCBase, WCBase>() {

        @Override
        public WCBase map(WCBase value) throws Exception {
            WC wc = (WC) value;
            wc.count = wc.secretCount;
            return wc;
        }
    });
    counts.writeAsText(resultPath);
    env.execute("WordCount with custom data types example");
}

Example 13 with ReduceFunction

use of org.apache.flink.api.common.functions.ReduceFunction in project flink by apache.

the class IterationWithAllReducerITCase method testProgram.

@Override
protected void testProgram() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(4);
    DataSet<String> initialInput = env.fromElements("1", "1", "1", "1", "1", "1", "1", "1");
    IterativeDataSet<String> iteration = initialInput.iterate(5).name("Loop");
    DataSet<String> sumReduce = iteration.reduce(new ReduceFunction<String>() {

        @Override
        public String reduce(String value1, String value2) throws Exception {
            return value1;
        }
    }).name("Compute sum (Reduce)");
    List<String> result = iteration.closeWith(sumReduce).collect();
    compareResultAsText(result, EXPECTED);
}

Example 14 with ReduceFunction

use of org.apache.flink.api.common.functions.ReduceFunction in project flink by apache.

the class TimestampITCase method testErrorOnEventTimeWithoutTimestamps.

@Test
public void testErrorOnEventTimeWithoutTimestamps() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
    env.setParallelism(2);
    env.getConfig().disableSysoutLogging();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    DataStream<Tuple2<String, Integer>> source1 = env.fromElements(new Tuple2<>("a", 1), new Tuple2<>("b", 2));
    source1.keyBy(0).window(TumblingEventTimeWindows.of(Time.seconds(5))).reduce(new ReduceFunction<Tuple2<String, Integer>>() {

        @Override
        public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) {
            return value1;
        }
    }).print();
    try {
        env.execute();
        fail("this should fail with an exception");
    } catch (Exception e) {
    // expected
    }
}

Example 15 with ReduceFunction

use of org.apache.flink.api.common.functions.ReduceFunction in project flink by apache.

the class StateBackendTestBase method testConcurrentMapIfQueryable.

/**
	 * Tests that {@link AbstractHeapState} instances respect the queryable
	 * flag and create concurrent variants for internal state structures.
	 */
@SuppressWarnings("unchecked")
protected void testConcurrentMapIfQueryable() throws Exception {
    final int numberOfKeyGroups = 1;
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, numberOfKeyGroups, new KeyGroupRange(0, 0), new DummyEnvironment("test_op", 1, 0));
    {
        // ValueState
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("value-state", Integer.class, -1);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.update(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // ListState
        ListStateDescriptor<Integer> desc = new ListStateDescriptor<>("list-state", Integer.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        ListState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.add(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // ReducingState
        ReducingStateDescriptor<Integer> desc = new ReducingStateDescriptor<>("reducing-state", new ReduceFunction<Integer>() {

            @Override
            public Integer reduce(Integer value1, Integer value2) throws Exception {
                return value1 + value2;
            }
        }, Integer.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        ReducingState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.add(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // FoldingState
        FoldingStateDescriptor<Integer, Integer> desc = new FoldingStateDescriptor<>("folding-state", 0, new FoldFunction<Integer, Integer>() {

            @Override
            public Integer fold(Integer accumulator, Integer value) throws Exception {
                return accumulator + value;
            }
        }, Integer.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        FoldingState<Integer, Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.add(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // MapState
        MapStateDescriptor<Integer, String> desc = new MapStateDescriptor<>("map-state", Integer.class, String.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        MapState<Integer, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.put(121818273, "121818273");
        int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(1, numberOfKeyGroups);
        StateTable stateTable = ((AbstractHeapState) kvState).getStateTable();
        assertNotNull("State not set", stateTable.get(keyGroupIndex));
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    backend.dispose();
}