Examples with ExecutionConfig org.apache.flink.api.common.ExecutionConfig used on opensource projects

Example 41 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class AggregationFunctionTest method groupSumIntegerTest.

@Test
public void groupSumIntegerTest() throws Exception {
    // preparing expected outputs
    List<Tuple2<Integer, Integer>> expectedGroupSumList = new ArrayList<>();
    List<Tuple2<Integer, Integer>> expectedGroupMinList = new ArrayList<>();
    List<Tuple2<Integer, Integer>> expectedGroupMaxList = new ArrayList<>();
    int groupedSum0 = 0;
    int groupedSum1 = 0;
    int groupedSum2 = 0;
    for (int i = 0; i < 9; i++) {
        int groupedSum;
        switch(i % 3) {
            case 0:
                groupedSum = groupedSum0 += i;
                break;
            case 1:
                groupedSum = groupedSum1 += i;
                break;
            default:
                groupedSum = groupedSum2 += i;
                break;
        }
        expectedGroupSumList.add(new Tuple2<>(i % 3, groupedSum));
        expectedGroupMinList.add(new Tuple2<>(i % 3, i % 3));
        expectedGroupMaxList.add(new Tuple2<>(i % 3, i));
    }
    // some necessary boiler plate
    TypeInformation<Tuple2<Integer, Integer>> typeInfo = TypeExtractor.getForObject(new Tuple2<>(0, 0));
    ExecutionConfig config = new ExecutionConfig();
    KeySelector<Tuple2<Integer, Integer>, Tuple> keySelector = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<>(new int[] { 0 }, typeInfo), typeInfo, config);
    TypeInformation<Tuple> keyType = TypeExtractor.getKeySelectorTypes(keySelector, typeInfo);
    // aggregations tested
    ReduceFunction<Tuple2<Integer, Integer>> sumFunction = new SumAggregator<>(1, typeInfo, config);
    ReduceFunction<Tuple2<Integer, Integer>> minFunction = new ComparableAggregator<>(1, typeInfo, AggregationType.MIN, config);
    ReduceFunction<Tuple2<Integer, Integer>> maxFunction = new ComparableAggregator<>(1, typeInfo, AggregationType.MAX, config);
    List<Tuple2<Integer, Integer>> groupedSumList = MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(sumFunction, typeInfo.createSerializer(config)), getInputList(), keySelector, keyType);
    List<Tuple2<Integer, Integer>> groupedMinList = MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(minFunction, typeInfo.createSerializer(config)), getInputList(), keySelector, keyType);
    List<Tuple2<Integer, Integer>> groupedMaxList = MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(maxFunction, typeInfo.createSerializer(config)), getInputList(), keySelector, keyType);
    assertEquals(expectedGroupSumList, groupedSumList);
    assertEquals(expectedGroupMinList, groupedMinList);
    assertEquals(expectedGroupMaxList, groupedMaxList);
}

Example 42 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class AggregationFunctionTest method pojoMinMaxByTest.

@Test
public void pojoMinMaxByTest() throws Exception {
    // Pojos are grouped on field 0, aggregated on field 1
    // preparing expected outputs
    List<MyPojo3> maxByFirstExpected = ImmutableList.of(new MyPojo3(0, 0), new MyPojo3(1, 1), new MyPojo3(2, 2), new MyPojo3(2, 2), new MyPojo3(2, 2), new MyPojo3(2, 2), new MyPojo3(2, 2), new MyPojo3(2, 2), new MyPojo3(2, 2));
    List<MyPojo3> maxByLastExpected = ImmutableList.of(new MyPojo3(0, 0), new MyPojo3(1, 1), new MyPojo3(2, 2), new MyPojo3(2, 2), new MyPojo3(2, 2), new MyPojo3(2, 5), new MyPojo3(2, 5), new MyPojo3(2, 5), new MyPojo3(2, 8));
    List<MyPojo3> minByFirstExpected = ImmutableList.of(new MyPojo3(0, 0), new MyPojo3(0, 0), new MyPojo3(0, 0), new MyPojo3(0, 0), new MyPojo3(0, 0), new MyPojo3(0, 0), new MyPojo3(0, 0), new MyPojo3(0, 0), new MyPojo3(0, 0));
    List<MyPojo3> minByLastExpected = ImmutableList.of(new MyPojo3(0, 0), new MyPojo3(0, 0), new MyPojo3(0, 0), new MyPojo3(0, 3), new MyPojo3(0, 3), new MyPojo3(0, 3), new MyPojo3(0, 6), new MyPojo3(0, 6), new MyPojo3(0, 6));
    // some necessary boiler plate
    TypeInformation<MyPojo3> typeInfo = TypeExtractor.getForObject(new MyPojo3(0, 0));
    ExecutionConfig config = new ExecutionConfig();
    KeySelector<MyPojo3, Tuple> keySelector = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<>(new String[] { "f0" }, typeInfo), typeInfo, config);
    TypeInformation<Tuple> keyType = TypeExtractor.getKeySelectorTypes(keySelector, typeInfo);
    // aggregations tested
    ReduceFunction<MyPojo3> maxByFunctionFirst = new ComparableAggregator<>("f1", typeInfo, AggregationType.MAXBY, true, config);
    ReduceFunction<MyPojo3> maxByFunctionLast = new ComparableAggregator<>("f1", typeInfo, AggregationType.MAXBY, false, config);
    ReduceFunction<MyPojo3> minByFunctionFirst = new ComparableAggregator<>("f1", typeInfo, AggregationType.MINBY, true, config);
    ReduceFunction<MyPojo3> minByFunctionLast = new ComparableAggregator<>("f1", typeInfo, AggregationType.MINBY, false, config);
    assertEquals(maxByFirstExpected, MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(maxByFunctionFirst, typeInfo.createSerializer(config)), getInputByPojoList(), keySelector, keyType));
    assertEquals(maxByLastExpected, MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(maxByFunctionLast, typeInfo.createSerializer(config)), getInputByPojoList(), keySelector, keyType));
    assertEquals(minByLastExpected, MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(minByFunctionLast, typeInfo.createSerializer(config)), getInputByPojoList(), keySelector, keyType));
    assertEquals(minByFirstExpected, MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(minByFunctionFirst, typeInfo.createSerializer(config)), getInputByPojoList(), keySelector, keyType));
}

Example 43 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class AggregationFunctionTest method pojoGroupSumIntegerTest.

@Test
public void pojoGroupSumIntegerTest() throws Exception {
    // preparing expected outputs
    List<MyPojo> expectedGroupSumList = new ArrayList<>();
    List<MyPojo> expectedGroupMinList = new ArrayList<>();
    List<MyPojo> expectedGroupMaxList = new ArrayList<>();
    int groupedSum0 = 0;
    int groupedSum1 = 0;
    int groupedSum2 = 0;
    for (int i = 0; i < 9; i++) {
        int groupedSum;
        switch(i % 3) {
            case 0:
                groupedSum = groupedSum0 += i;
                break;
            case 1:
                groupedSum = groupedSum1 += i;
                break;
            default:
                groupedSum = groupedSum2 += i;
                break;
        }
        expectedGroupSumList.add(new MyPojo(i % 3, groupedSum));
        expectedGroupMinList.add(new MyPojo(i % 3, i % 3));
        expectedGroupMaxList.add(new MyPojo(i % 3, i));
    }
    // some necessary boiler plate
    TypeInformation<MyPojo> typeInfo = TypeExtractor.getForObject(new MyPojo(0, 0));
    ExecutionConfig config = new ExecutionConfig();
    KeySelector<MyPojo, Tuple> keySelector = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<>(new String[] { "f0" }, typeInfo), typeInfo, config);
    TypeInformation<Tuple> keyType = TypeExtractor.getKeySelectorTypes(keySelector, typeInfo);
    // aggregations tested
    ReduceFunction<MyPojo> sumFunction = new SumAggregator<>("f1", typeInfo, config);
    ReduceFunction<MyPojo> minFunction = new ComparableAggregator<>("f1", typeInfo, AggregationType.MIN, false, config);
    ReduceFunction<MyPojo> maxFunction = new ComparableAggregator<>("f1", typeInfo, AggregationType.MAX, false, config);
    List<MyPojo> groupedSumList = MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(sumFunction, typeInfo.createSerializer(config)), getInputPojoList(), keySelector, keyType);
    List<MyPojo> groupedMinList = MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(minFunction, typeInfo.createSerializer(config)), getInputPojoList(), keySelector, keyType);
    List<MyPojo> groupedMaxList = MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(maxFunction, typeInfo.createSerializer(config)), getInputPojoList(), keySelector, keyType);
    assertEquals(expectedGroupSumList, groupedSumList);
    assertEquals(expectedGroupMinList, groupedMinList);
    assertEquals(expectedGroupMaxList, groupedMaxList);
}

Example 44 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class AggregationFunctionTest method minMaxByTest.

@Test
public void minMaxByTest() throws Exception {
    // Tuples are grouped on field 0, aggregated on field 1
    // preparing expected outputs
    List<Tuple3<Integer, Integer, Integer>> maxByFirstExpected = ImmutableList.of(Tuple3.of(0, 0, 0), Tuple3.of(0, 1, 1), Tuple3.of(0, 2, 2), Tuple3.of(0, 2, 2), Tuple3.of(0, 2, 2), Tuple3.of(0, 2, 2), Tuple3.of(0, 2, 2), Tuple3.of(0, 2, 2), Tuple3.of(0, 2, 2));
    List<Tuple3<Integer, Integer, Integer>> maxByLastExpected = ImmutableList.of(Tuple3.of(0, 0, 0), Tuple3.of(0, 1, 1), Tuple3.of(0, 2, 2), Tuple3.of(0, 2, 2), Tuple3.of(0, 2, 2), Tuple3.of(0, 2, 5), Tuple3.of(0, 2, 5), Tuple3.of(0, 2, 5), Tuple3.of(0, 2, 8));
    List<Tuple3<Integer, Integer, Integer>> minByFirstExpected = ImmutableList.of(Tuple3.of(0, 0, 0), Tuple3.of(0, 0, 0), Tuple3.of(0, 0, 0), Tuple3.of(0, 0, 0), Tuple3.of(0, 0, 0), Tuple3.of(0, 0, 0), Tuple3.of(0, 0, 0), Tuple3.of(0, 0, 0), Tuple3.of(0, 0, 0));
    List<Tuple3<Integer, Integer, Integer>> minByLastExpected = ImmutableList.of(Tuple3.of(0, 0, 0), Tuple3.of(0, 0, 0), Tuple3.of(0, 0, 0), Tuple3.of(0, 0, 3), Tuple3.of(0, 0, 3), Tuple3.of(0, 0, 3), Tuple3.of(0, 0, 6), Tuple3.of(0, 0, 6), Tuple3.of(0, 0, 6));
    // some necessary boiler plate
    TypeInformation<Tuple3<Integer, Integer, Integer>> typeInfo = TypeExtractor.getForObject(Tuple3.of(0, 0, 0));
    ExecutionConfig config = new ExecutionConfig();
    KeySelector<Tuple3<Integer, Integer, Integer>, Tuple> keySelector = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<>(new int[] { 0 }, typeInfo), typeInfo, config);
    TypeInformation<Tuple> keyType = TypeExtractor.getKeySelectorTypes(keySelector, typeInfo);
    // aggregations tested
    ReduceFunction<Tuple3<Integer, Integer, Integer>> maxByFunctionFirst = new ComparableAggregator<>(1, typeInfo, AggregationType.MAXBY, true, config);
    ReduceFunction<Tuple3<Integer, Integer, Integer>> maxByFunctionLast = new ComparableAggregator<>(1, typeInfo, AggregationType.MAXBY, false, config);
    ReduceFunction<Tuple3<Integer, Integer, Integer>> minByFunctionFirst = new ComparableAggregator<>(1, typeInfo, AggregationType.MINBY, true, config);
    ReduceFunction<Tuple3<Integer, Integer, Integer>> minByFunctionLast = new ComparableAggregator<>(1, typeInfo, AggregationType.MINBY, false, config);
    assertEquals(maxByFirstExpected, MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(maxByFunctionFirst, typeInfo.createSerializer(config)), getInputByList(), keySelector, keyType));
    assertEquals(maxByLastExpected, MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(maxByFunctionLast, typeInfo.createSerializer(config)), getInputByList(), keySelector, keyType));
    assertEquals(minByLastExpected, MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(minByFunctionLast, typeInfo.createSerializer(config)), getInputByList(), keySelector, keyType));
    assertEquals(minByFirstExpected, MockContext.createAndExecuteForKeyedStream(new StreamGroupedReduce<>(minByFunctionFirst, typeInfo.createSerializer(config)), getInputByList(), keySelector, keyType));
}

Example 45 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class RichAsyncFunctionTest method testRuntimeContext.

/**
	 * Test the set of runtime context methods in the context of a {@link RichAsyncFunction}.
	 */
@Test
public void testRuntimeContext() throws Exception {
    RichAsyncFunction<Integer, Integer> function = new RichAsyncFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1707630162838967972L;

        @Override
        public void asyncInvoke(Integer input, AsyncCollector<Integer> collector) throws Exception {
        // no op
        }
    };
    final String taskName = "foobarTask";
    final MetricGroup metricGroup = mock(MetricGroup.class);
    final int numberOfParallelSubtasks = 42;
    final int indexOfSubtask = 43;
    final int attemptNumber = 1337;
    final String taskNameWithSubtask = "barfoo";
    final ExecutionConfig executionConfig = mock(ExecutionConfig.class);
    final ClassLoader userCodeClassLoader = mock(ClassLoader.class);
    RuntimeContext mockedRuntimeContext = mock(RuntimeContext.class);
    when(mockedRuntimeContext.getTaskName()).thenReturn(taskName);
    when(mockedRuntimeContext.getMetricGroup()).thenReturn(metricGroup);
    when(mockedRuntimeContext.getNumberOfParallelSubtasks()).thenReturn(numberOfParallelSubtasks);
    when(mockedRuntimeContext.getIndexOfThisSubtask()).thenReturn(indexOfSubtask);
    when(mockedRuntimeContext.getAttemptNumber()).thenReturn(attemptNumber);
    when(mockedRuntimeContext.getTaskNameWithSubtasks()).thenReturn(taskNameWithSubtask);
    when(mockedRuntimeContext.getExecutionConfig()).thenReturn(executionConfig);
    when(mockedRuntimeContext.getUserCodeClassLoader()).thenReturn(userCodeClassLoader);
    function.setRuntimeContext(mockedRuntimeContext);
    RuntimeContext runtimeContext = function.getRuntimeContext();
    assertEquals(taskName, runtimeContext.getTaskName());
    assertEquals(metricGroup, runtimeContext.getMetricGroup());
    assertEquals(numberOfParallelSubtasks, runtimeContext.getNumberOfParallelSubtasks());
    assertEquals(indexOfSubtask, runtimeContext.getIndexOfThisSubtask());
    assertEquals(attemptNumber, runtimeContext.getAttemptNumber());
    assertEquals(taskNameWithSubtask, runtimeContext.getTaskNameWithSubtasks());
    assertEquals(executionConfig, runtimeContext.getExecutionConfig());
    assertEquals(userCodeClassLoader, runtimeContext.getUserCodeClassLoader());
    try {
        runtimeContext.getDistributedCache();
        fail("Expected getDistributedCached to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getState(new ValueStateDescriptor<>("foobar", Integer.class, 42));
        fail("Expected getState to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getListState(new ListStateDescriptor<>("foobar", Integer.class));
        fail("Expected getListState to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getReducingState(new ReducingStateDescriptor<>("foobar", new ReduceFunction<Integer>() {

            private static final long serialVersionUID = 2136425961884441050L;

            @Override
            public Integer reduce(Integer value1, Integer value2) throws Exception {
                return value1;
            }
        }, Integer.class));
        fail("Expected getReducingState to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getFoldingState(new FoldingStateDescriptor<>("foobar", 0, new FoldFunction<Integer, Integer>() {

            @Override
            public Integer fold(Integer accumulator, Integer value) throws Exception {
                return accumulator;
            }
        }, Integer.class));
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getMapState(new MapStateDescriptor<>("foobar", Integer.class, String.class));
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.addAccumulator("foobar", new Accumulator<Integer, Integer>() {

            private static final long serialVersionUID = -4673320336846482358L;

            @Override
            public void add(Integer value) {
            // no op
            }

            @Override
            public Integer getLocalValue() {
                return null;
            }

            @Override
            public void resetLocal() {
            }

            @Override
            public void merge(Accumulator<Integer, Integer> other) {
            }

            @Override
            public Accumulator<Integer, Integer> clone() {
                return null;
            }
        });
        fail("Expected addAccumulator to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getAccumulator("foobar");
        fail("Expected getAccumulator to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getAllAccumulators();
        fail("Expected getAllAccumulators to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getIntCounter("foobar");
        fail("Expected getIntCounter to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getLongCounter("foobar");
        fail("Expected getLongCounter to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getDoubleCounter("foobar");
        fail("Expected getDoubleCounter to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getHistogram("foobar");
        fail("Expected getHistogram to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.hasBroadcastVariable("foobar");
        fail("Expected hasBroadcastVariable to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getBroadcastVariable("foobar");
        fail("Expected getBroadcastVariable to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getBroadcastVariableWithInitializer("foobar", new BroadcastVariableInitializer<Object, Object>() {

            @Override
            public Object initializeBroadcastVariable(Iterable<Object> data) {
                return null;
            }
        });
        fail("Expected getBroadcastVariableWithInitializer to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
}