Examples with CollectorOutput org.apache.flink.streaming.util.CollectorOutput used on opensource projects

Example 1 with CollectorOutput

use of org.apache.flink.streaming.util.CollectorOutput in project flink by apache.

the class EmitterTest method testEmitterWithExceptions.

/**
	 * Tests that the emitter handles exceptions occurring in the {@link AsyncCollector} correctly.
	 */
@Test
public void testEmitterWithExceptions() throws Exception {
    Object lock = new Object();
    List<StreamElement> list = new ArrayList<>();
    Output<StreamRecord<Integer>> output = new CollectorOutput<>(list);
    List<StreamElement> expected = Arrays.asList(new StreamRecord<>(1, 0L), new Watermark(3L));
    OperatorActions operatorActions = mock(OperatorActions.class);
    final int capacity = 3;
    StreamElementQueue queue = new OrderedStreamElementQueue(capacity, executor, operatorActions);
    final Emitter<Integer> emitter = new Emitter<>(lock, output, queue, operatorActions);
    final Thread emitterThread = new Thread(emitter);
    emitterThread.start();
    final Exception testException = new Exception("Test exception");
    try {
        StreamRecordQueueEntry<Integer> record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L));
        StreamRecordQueueEntry<Integer> record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L));
        WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(3L));
        queue.put(record1);
        queue.put(record2);
        queue.put(watermark1);
        record2.collect(testException);
        record1.collect(Arrays.asList(1));
        synchronized (lock) {
            while (!queue.isEmpty()) {
                lock.wait();
            }
        }
        Assert.assertEquals(expected, list);
        ArgumentCaptor<Throwable> argumentCaptor = ArgumentCaptor.forClass(Throwable.class);
        verify(operatorActions).failOperator(argumentCaptor.capture());
        Throwable failureCause = argumentCaptor.getValue();
        Assert.assertNotNull(failureCause.getCause());
        Assert.assertTrue(failureCause.getCause() instanceof ExecutionException);
        Assert.assertNotNull(failureCause.getCause().getCause());
        Assert.assertEquals(testException, failureCause.getCause().getCause());
    } finally {
        emitter.stop();
        emitterThread.interrupt();
    }
}

Example 2 with CollectorOutput

use of org.apache.flink.streaming.util.CollectorOutput in project flink by apache.

the class EmitterTest method testEmitterWithOrderedQueue.

/**
	 * Tests that the emitter outputs completed stream element queue entries.
	 */
@Test
public void testEmitterWithOrderedQueue() throws Exception {
    Object lock = new Object();
    List<StreamElement> list = new ArrayList<>();
    Output<StreamRecord<Integer>> output = new CollectorOutput<>(list);
    List<StreamElement> expected = Arrays.asList(new StreamRecord<>(1, 0L), new StreamRecord<>(2, 0L), new StreamRecord<>(3, 1L), new StreamRecord<>(4, 1L), new Watermark(3L), new StreamRecord<>(5, 4L), new StreamRecord<>(6, 4L));
    OperatorActions operatorActions = mock(OperatorActions.class);
    final int capacity = 5;
    StreamElementQueue queue = new OrderedStreamElementQueue(capacity, executor, operatorActions);
    final Emitter<Integer> emitter = new Emitter<>(lock, output, queue, operatorActions);
    final Thread emitterThread = new Thread(emitter);
    emitterThread.start();
    try {
        StreamRecordQueueEntry<Integer> record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L));
        StreamRecordQueueEntry<Integer> record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L));
        WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(3L));
        StreamRecordQueueEntry<Integer> record3 = new StreamRecordQueueEntry<>(new StreamRecord<>(3, 4L));
        queue.put(record1);
        queue.put(record2);
        queue.put(watermark1);
        queue.put(record3);
        record2.collect(Arrays.asList(3, 4));
        record1.collect(Arrays.asList(1, 2));
        record3.collect(Arrays.asList(5, 6));
        synchronized (lock) {
            while (!queue.isEmpty()) {
                lock.wait();
            }
        }
        Assert.assertEquals(expected, list);
    } finally {
        emitter.stop();
        emitterThread.interrupt();
    }
}

Example 3 with CollectorOutput

use of org.apache.flink.streaming.util.CollectorOutput in project flink by apache.

the class StreamingRuntimeContextTest method createDescriptorCapturingMockOp.

@SuppressWarnings("unchecked")
private static AbstractStreamOperator<?> createDescriptorCapturingMockOp(final AtomicReference<Object> ref, final ExecutionConfig config, Environment environment) throws Exception {
    AbstractStreamOperator<?> operator = new AbstractStreamOperator<Object>() {

        @Override
        public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<Object>> output) {
            super.setup(containingTask, config, output);
        }
    };
    StreamConfig streamConfig = new StreamConfig(new Configuration());
    streamConfig.setOperatorID(new OperatorID());
    operator.setup(new MockStreamTaskBuilder(environment).setExecutionConfig(config).build(), streamConfig, new CollectorOutput<>(new ArrayList<>()));
    StreamTaskStateInitializer streamTaskStateManager = new StreamTaskStateInitializerImpl(environment, new MemoryStateBackend());
    KeyedStateBackend keyedStateBackend = mock(KeyedStateBackend.class);
    DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config);
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ref.set(invocationOnMock.getArguments()[2]);
            return null;
        }
    }).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(StateDescriptor.class));
    operator.initializeState(streamTaskStateManager);
    operator.getRuntimeContext().setKeyedStateStore(keyedStateStore);
    return operator;
}