Examples with AsyncCollector org.apache.flink.streaming.api.functions.async.collector.AsyncCollector used on opensource projects

Example 1 with AsyncCollector

use of org.apache.flink.streaming.api.functions.async.collector.AsyncCollector in project flink by apache.

the class RichAsyncFunctionTest method testRuntimeContext.

/**
	 * Test the set of runtime context methods in the context of a {@link RichAsyncFunction}.
	 */
@Test
public void testRuntimeContext() throws Exception {
    RichAsyncFunction<Integer, Integer> function = new RichAsyncFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1707630162838967972L;

        @Override
        public void asyncInvoke(Integer input, AsyncCollector<Integer> collector) throws Exception {
        // no op
        }
    };
    final String taskName = "foobarTask";
    final MetricGroup metricGroup = mock(MetricGroup.class);
    final int numberOfParallelSubtasks = 42;
    final int indexOfSubtask = 43;
    final int attemptNumber = 1337;
    final String taskNameWithSubtask = "barfoo";
    final ExecutionConfig executionConfig = mock(ExecutionConfig.class);
    final ClassLoader userCodeClassLoader = mock(ClassLoader.class);
    RuntimeContext mockedRuntimeContext = mock(RuntimeContext.class);
    when(mockedRuntimeContext.getTaskName()).thenReturn(taskName);
    when(mockedRuntimeContext.getMetricGroup()).thenReturn(metricGroup);
    when(mockedRuntimeContext.getNumberOfParallelSubtasks()).thenReturn(numberOfParallelSubtasks);
    when(mockedRuntimeContext.getIndexOfThisSubtask()).thenReturn(indexOfSubtask);
    when(mockedRuntimeContext.getAttemptNumber()).thenReturn(attemptNumber);
    when(mockedRuntimeContext.getTaskNameWithSubtasks()).thenReturn(taskNameWithSubtask);
    when(mockedRuntimeContext.getExecutionConfig()).thenReturn(executionConfig);
    when(mockedRuntimeContext.getUserCodeClassLoader()).thenReturn(userCodeClassLoader);
    function.setRuntimeContext(mockedRuntimeContext);
    RuntimeContext runtimeContext = function.getRuntimeContext();
    assertEquals(taskName, runtimeContext.getTaskName());
    assertEquals(metricGroup, runtimeContext.getMetricGroup());
    assertEquals(numberOfParallelSubtasks, runtimeContext.getNumberOfParallelSubtasks());
    assertEquals(indexOfSubtask, runtimeContext.getIndexOfThisSubtask());
    assertEquals(attemptNumber, runtimeContext.getAttemptNumber());
    assertEquals(taskNameWithSubtask, runtimeContext.getTaskNameWithSubtasks());
    assertEquals(executionConfig, runtimeContext.getExecutionConfig());
    assertEquals(userCodeClassLoader, runtimeContext.getUserCodeClassLoader());
    try {
        runtimeContext.getDistributedCache();
        fail("Expected getDistributedCached to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getState(new ValueStateDescriptor<>("foobar", Integer.class, 42));
        fail("Expected getState to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getListState(new ListStateDescriptor<>("foobar", Integer.class));
        fail("Expected getListState to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getReducingState(new ReducingStateDescriptor<>("foobar", new ReduceFunction<Integer>() {

            private static final long serialVersionUID = 2136425961884441050L;

            @Override
            public Integer reduce(Integer value1, Integer value2) throws Exception {
                return value1;
            }
        }, Integer.class));
        fail("Expected getReducingState to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getFoldingState(new FoldingStateDescriptor<>("foobar", 0, new FoldFunction<Integer, Integer>() {

            @Override
            public Integer fold(Integer accumulator, Integer value) throws Exception {
                return accumulator;
            }
        }, Integer.class));
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getMapState(new MapStateDescriptor<>("foobar", Integer.class, String.class));
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.addAccumulator("foobar", new Accumulator<Integer, Integer>() {

            private static final long serialVersionUID = -4673320336846482358L;

            @Override
            public void add(Integer value) {
            // no op
            }

            @Override
            public Integer getLocalValue() {
                return null;
            }

            @Override
            public void resetLocal() {
            }

            @Override
            public void merge(Accumulator<Integer, Integer> other) {
            }

            @Override
            public Accumulator<Integer, Integer> clone() {
                return null;
            }
        });
        fail("Expected addAccumulator to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getAccumulator("foobar");
        fail("Expected getAccumulator to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getAllAccumulators();
        fail("Expected getAllAccumulators to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getIntCounter("foobar");
        fail("Expected getIntCounter to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getLongCounter("foobar");
        fail("Expected getLongCounter to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getDoubleCounter("foobar");
        fail("Expected getDoubleCounter to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getHistogram("foobar");
        fail("Expected getHistogram to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.hasBroadcastVariable("foobar");
        fail("Expected hasBroadcastVariable to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getBroadcastVariable("foobar");
        fail("Expected getBroadcastVariable to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        runtimeContext.getBroadcastVariableWithInitializer("foobar", new BroadcastVariableInitializer<Object, Object>() {

            @Override
            public Object initializeBroadcastVariable(Iterable<Object> data) {
                return null;
            }
        });
        fail("Expected getBroadcastVariableWithInitializer to fail with unsupported operation exception.");
    } catch (UnsupportedOperationException e) {
    // expected
    }
}

Example 2 with AsyncCollector

use of org.apache.flink.streaming.api.functions.async.collector.AsyncCollector in project flink by apache.

the class RichAsyncFunctionTest method testIterationRuntimeContext.

/**
	 * Test the set of iteration runtime context methods in the context of a
	 * {@link RichAsyncFunction}.
	 */
@Test
public void testIterationRuntimeContext() throws Exception {
    RichAsyncFunction<Integer, Integer> function = new RichAsyncFunction<Integer, Integer>() {

        private static final long serialVersionUID = -2023923961609455894L;

        @Override
        public void asyncInvoke(Integer input, AsyncCollector<Integer> collector) throws Exception {
        // no op
        }
    };
    int superstepNumber = 42;
    IterationRuntimeContext mockedIterationRuntimeContext = mock(IterationRuntimeContext.class);
    when(mockedIterationRuntimeContext.getSuperstepNumber()).thenReturn(superstepNumber);
    function.setRuntimeContext(mockedIterationRuntimeContext);
    IterationRuntimeContext iterationRuntimeContext = function.getIterationRuntimeContext();
    assertEquals(superstepNumber, iterationRuntimeContext.getSuperstepNumber());
    try {
        iterationRuntimeContext.getIterationAggregator("foobar");
        fail("Expected getIterationAggregator to fail with unsupported operation exception");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        iterationRuntimeContext.getPreviousIterationAggregate("foobar");
        fail("Expected getPreviousIterationAggregator to fail with unsupported operation exception");
    } catch (UnsupportedOperationException e) {
    // expected
    }
}

Example 3 with AsyncCollector

use of org.apache.flink.streaming.api.functions.async.collector.AsyncCollector in project flink by apache.

the class StreamingOperatorsITCase method testAsyncWaitOperator.

/**
	 * Tests the basic functionality of the AsyncWaitOperator: Processing a limited stream of
	 * elements by doubling their value. This is tested in for the ordered and unordered mode.
	 */
@Test
public void testAsyncWaitOperator() throws Exception {
    final int numElements = 5;
    final long timeout = 1000L;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Tuple2<Integer, NonSerializable>> input = env.addSource(new NonSerializableTupleSource(numElements));
    AsyncFunction<Tuple2<Integer, NonSerializable>, Integer> function = new RichAsyncFunction<Tuple2<Integer, NonSerializable>, Integer>() {

        private static final long serialVersionUID = 7000343199829487985L;

        transient ExecutorService executorService;

        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
            executorService = Executors.newFixedThreadPool(numElements);
        }

        @Override
        public void close() throws Exception {
            super.close();
            executorService.shutdownNow();
        }

        @Override
        public void asyncInvoke(final Tuple2<Integer, NonSerializable> input, final AsyncCollector<Integer> collector) throws Exception {
            executorService.submit(new Runnable() {

                @Override
                public void run() {
                    collector.collect(Collections.singletonList(input.f0 + input.f0));
                }
            });
        }
    };
    DataStream<Integer> orderedResult = AsyncDataStream.orderedWait(input, function, timeout, TimeUnit.MILLISECONDS, 2).setParallelism(1);
    // save result from ordered process
    final MemorySinkFunction sinkFunction1 = new MemorySinkFunction(0);
    final List<Integer> actualResult1 = new ArrayList<>(numElements);
    MemorySinkFunction.registerCollection(0, actualResult1);
    orderedResult.addSink(sinkFunction1).setParallelism(1);
    DataStream<Integer> unorderedResult = AsyncDataStream.unorderedWait(input, function, timeout, TimeUnit.MILLISECONDS, 2);
    // save result from unordered process
    final MemorySinkFunction sinkFunction2 = new MemorySinkFunction(1);
    final List<Integer> actualResult2 = new ArrayList<>(numElements);
    MemorySinkFunction.registerCollection(1, actualResult2);
    unorderedResult.addSink(sinkFunction2);
    Collection<Integer> expected = new ArrayList<>(10);
    for (int i = 0; i < numElements; i++) {
        expected.add(i + i);
    }
    env.execute();
    Assert.assertEquals(expected, actualResult1);
    Collections.sort(actualResult2);
    Assert.assertEquals(expected, actualResult2);
    MemorySinkFunction.clear();
}