Examples with StreamRecordQueueEntry org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry used on opensource projects

Example 1 with StreamRecordQueueEntry

use of org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry in project flink by apache.

the class AsyncWaitOperator method processElement.

@Override
public void processElement(StreamRecord<IN> element) throws Exception {
    final StreamRecordQueueEntry<OUT> streamRecordBufferEntry = new StreamRecordQueueEntry<>(element);
    if (timeout > 0L) {
        // register a timeout for this AsyncStreamRecordBufferEntry
        long timeoutTimestamp = timeout + getProcessingTimeService().getCurrentProcessingTime();
        final ScheduledFuture<?> timerFuture = getProcessingTimeService().registerTimer(timeoutTimestamp, new ProcessingTimeCallback() {

            @Override
            public void onProcessingTime(long timestamp) throws Exception {
                streamRecordBufferEntry.collect(new TimeoutException("Async function call has timed out."));
            }
        });
        // Cancel the timer once we've completed the stream record buffer entry. This will remove
        // the register trigger task
        streamRecordBufferEntry.onComplete(new AcceptFunction<StreamElementQueueEntry<Collection<OUT>>>() {

            @Override
            public void accept(StreamElementQueueEntry<Collection<OUT>> value) {
                timerFuture.cancel(true);
            }
        }, executor);
    }
    addAsyncBufferEntry(streamRecordBufferEntry);
    userFunction.asyncInvoke(element.getValue(), streamRecordBufferEntry);
}

Example 2 with StreamRecordQueueEntry

use of org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry in project flink by apache.

the class EmitterTest method testEmitterWithExceptions.

/**
	 * Tests that the emitter handles exceptions occurring in the {@link AsyncCollector} correctly.
	 */
@Test
public void testEmitterWithExceptions() throws Exception {
    Object lock = new Object();
    List<StreamElement> list = new ArrayList<>();
    Output<StreamRecord<Integer>> output = new CollectorOutput<>(list);
    List<StreamElement> expected = Arrays.asList(new StreamRecord<>(1, 0L), new Watermark(3L));
    OperatorActions operatorActions = mock(OperatorActions.class);
    final int capacity = 3;
    StreamElementQueue queue = new OrderedStreamElementQueue(capacity, executor, operatorActions);
    final Emitter<Integer> emitter = new Emitter<>(lock, output, queue, operatorActions);
    final Thread emitterThread = new Thread(emitter);
    emitterThread.start();
    final Exception testException = new Exception("Test exception");
    try {
        StreamRecordQueueEntry<Integer> record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L));
        StreamRecordQueueEntry<Integer> record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L));
        WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(3L));
        queue.put(record1);
        queue.put(record2);
        queue.put(watermark1);
        record2.collect(testException);
        record1.collect(Arrays.asList(1));
        synchronized (lock) {
            while (!queue.isEmpty()) {
                lock.wait();
            }
        }
        Assert.assertEquals(expected, list);
        ArgumentCaptor<Throwable> argumentCaptor = ArgumentCaptor.forClass(Throwable.class);
        verify(operatorActions).failOperator(argumentCaptor.capture());
        Throwable failureCause = argumentCaptor.getValue();
        Assert.assertNotNull(failureCause.getCause());
        Assert.assertTrue(failureCause.getCause() instanceof ExecutionException);
        Assert.assertNotNull(failureCause.getCause().getCause());
        Assert.assertEquals(testException, failureCause.getCause().getCause());
    } finally {
        emitter.stop();
        emitterThread.interrupt();
    }
}

Example 3 with StreamRecordQueueEntry

use of org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry in project flink by apache.

the class EmitterTest method testEmitterWithOrderedQueue.

/**
	 * Tests that the emitter outputs completed stream element queue entries.
	 */
@Test
public void testEmitterWithOrderedQueue() throws Exception {
    Object lock = new Object();
    List<StreamElement> list = new ArrayList<>();
    Output<StreamRecord<Integer>> output = new CollectorOutput<>(list);
    List<StreamElement> expected = Arrays.asList(new StreamRecord<>(1, 0L), new StreamRecord<>(2, 0L), new StreamRecord<>(3, 1L), new StreamRecord<>(4, 1L), new Watermark(3L), new StreamRecord<>(5, 4L), new StreamRecord<>(6, 4L));
    OperatorActions operatorActions = mock(OperatorActions.class);
    final int capacity = 5;
    StreamElementQueue queue = new OrderedStreamElementQueue(capacity, executor, operatorActions);
    final Emitter<Integer> emitter = new Emitter<>(lock, output, queue, operatorActions);
    final Thread emitterThread = new Thread(emitter);
    emitterThread.start();
    try {
        StreamRecordQueueEntry<Integer> record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L));
        StreamRecordQueueEntry<Integer> record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L));
        WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(3L));
        StreamRecordQueueEntry<Integer> record3 = new StreamRecordQueueEntry<>(new StreamRecord<>(3, 4L));
        queue.put(record1);
        queue.put(record2);
        queue.put(watermark1);
        queue.put(record3);
        record2.collect(Arrays.asList(3, 4));
        record1.collect(Arrays.asList(1, 2));
        record3.collect(Arrays.asList(5, 6));
        synchronized (lock) {
            while (!queue.isEmpty()) {
                lock.wait();
            }
        }
        Assert.assertEquals(expected, list);
    } finally {
        emitter.stop();
        emitterThread.interrupt();
    }
}