Search in sources :

Example 1 with StreamRecordQueueEntry

use of org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry in project flink by apache.

the class AsyncWaitOperator method processElement.

@Override
public void processElement(StreamRecord<IN> element) throws Exception {
    final StreamRecordQueueEntry<OUT> streamRecordBufferEntry = new StreamRecordQueueEntry<>(element);
    if (timeout > 0L) {
        // register a timeout for this AsyncStreamRecordBufferEntry
        long timeoutTimestamp = timeout + getProcessingTimeService().getCurrentProcessingTime();
        final ScheduledFuture<?> timerFuture = getProcessingTimeService().registerTimer(timeoutTimestamp, new ProcessingTimeCallback() {

            @Override
            public void onProcessingTime(long timestamp) throws Exception {
                streamRecordBufferEntry.collect(new TimeoutException("Async function call has timed out."));
            }
        });
        // Cancel the timer once we've completed the stream record buffer entry. This will remove
        // the register trigger task
        streamRecordBufferEntry.onComplete(new AcceptFunction<StreamElementQueueEntry<Collection<OUT>>>() {

            @Override
            public void accept(StreamElementQueueEntry<Collection<OUT>> value) {
                timerFuture.cancel(true);
            }
        }, executor);
    }
    addAsyncBufferEntry(streamRecordBufferEntry);
    userFunction.asyncInvoke(element.getValue(), streamRecordBufferEntry);
}
Also used : StreamElementQueueEntry(org.apache.flink.streaming.api.operators.async.queue.StreamElementQueueEntry) TimeoutException(java.util.concurrent.TimeoutException) ProcessingTimeCallback(org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback) Collection(java.util.Collection) StreamRecordQueueEntry(org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry) TimeoutException(java.util.concurrent.TimeoutException)

Example 2 with StreamRecordQueueEntry

use of org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry in project flink by apache.

the class EmitterTest method testEmitterWithExceptions.

/**
	 * Tests that the emitter handles exceptions occurring in the {@link AsyncCollector} correctly.
	 */
@Test
public void testEmitterWithExceptions() throws Exception {
    Object lock = new Object();
    List<StreamElement> list = new ArrayList<>();
    Output<StreamRecord<Integer>> output = new CollectorOutput<>(list);
    List<StreamElement> expected = Arrays.asList(new StreamRecord<>(1, 0L), new Watermark(3L));
    OperatorActions operatorActions = mock(OperatorActions.class);
    final int capacity = 3;
    StreamElementQueue queue = new OrderedStreamElementQueue(capacity, executor, operatorActions);
    final Emitter<Integer> emitter = new Emitter<>(lock, output, queue, operatorActions);
    final Thread emitterThread = new Thread(emitter);
    emitterThread.start();
    final Exception testException = new Exception("Test exception");
    try {
        StreamRecordQueueEntry<Integer> record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L));
        StreamRecordQueueEntry<Integer> record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L));
        WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(3L));
        queue.put(record1);
        queue.put(record2);
        queue.put(watermark1);
        record2.collect(testException);
        record1.collect(Arrays.asList(1));
        synchronized (lock) {
            while (!queue.isEmpty()) {
                lock.wait();
            }
        }
        Assert.assertEquals(expected, list);
        ArgumentCaptor<Throwable> argumentCaptor = ArgumentCaptor.forClass(Throwable.class);
        verify(operatorActions).failOperator(argumentCaptor.capture());
        Throwable failureCause = argumentCaptor.getValue();
        Assert.assertNotNull(failureCause.getCause());
        Assert.assertTrue(failureCause.getCause() instanceof ExecutionException);
        Assert.assertNotNull(failureCause.getCause().getCause());
        Assert.assertEquals(testException, failureCause.getCause().getCause());
    } finally {
        emitter.stop();
        emitterThread.interrupt();
    }
}
Also used : StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) ArrayList(java.util.ArrayList) StreamElement(org.apache.flink.streaming.runtime.streamrecord.StreamElement) ExecutionException(java.util.concurrent.ExecutionException) StreamElementQueue(org.apache.flink.streaming.api.operators.async.queue.StreamElementQueue) OrderedStreamElementQueue(org.apache.flink.streaming.api.operators.async.queue.OrderedStreamElementQueue) OrderedStreamElementQueue(org.apache.flink.streaming.api.operators.async.queue.OrderedStreamElementQueue) CollectorOutput(org.apache.flink.streaming.util.CollectorOutput) ExecutionException(java.util.concurrent.ExecutionException) Watermark(org.apache.flink.streaming.api.watermark.Watermark) StreamRecordQueueEntry(org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry) WatermarkQueueEntry(org.apache.flink.streaming.api.operators.async.queue.WatermarkQueueEntry) Test(org.junit.Test)

Example 3 with StreamRecordQueueEntry

use of org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry in project flink by apache.

the class EmitterTest method testEmitterWithOrderedQueue.

/**
	 * Tests that the emitter outputs completed stream element queue entries.
	 */
@Test
public void testEmitterWithOrderedQueue() throws Exception {
    Object lock = new Object();
    List<StreamElement> list = new ArrayList<>();
    Output<StreamRecord<Integer>> output = new CollectorOutput<>(list);
    List<StreamElement> expected = Arrays.asList(new StreamRecord<>(1, 0L), new StreamRecord<>(2, 0L), new StreamRecord<>(3, 1L), new StreamRecord<>(4, 1L), new Watermark(3L), new StreamRecord<>(5, 4L), new StreamRecord<>(6, 4L));
    OperatorActions operatorActions = mock(OperatorActions.class);
    final int capacity = 5;
    StreamElementQueue queue = new OrderedStreamElementQueue(capacity, executor, operatorActions);
    final Emitter<Integer> emitter = new Emitter<>(lock, output, queue, operatorActions);
    final Thread emitterThread = new Thread(emitter);
    emitterThread.start();
    try {
        StreamRecordQueueEntry<Integer> record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L));
        StreamRecordQueueEntry<Integer> record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L));
        WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(3L));
        StreamRecordQueueEntry<Integer> record3 = new StreamRecordQueueEntry<>(new StreamRecord<>(3, 4L));
        queue.put(record1);
        queue.put(record2);
        queue.put(watermark1);
        queue.put(record3);
        record2.collect(Arrays.asList(3, 4));
        record1.collect(Arrays.asList(1, 2));
        record3.collect(Arrays.asList(5, 6));
        synchronized (lock) {
            while (!queue.isEmpty()) {
                lock.wait();
            }
        }
        Assert.assertEquals(expected, list);
    } finally {
        emitter.stop();
        emitterThread.interrupt();
    }
}
Also used : StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) ArrayList(java.util.ArrayList) StreamElement(org.apache.flink.streaming.runtime.streamrecord.StreamElement) StreamElementQueue(org.apache.flink.streaming.api.operators.async.queue.StreamElementQueue) OrderedStreamElementQueue(org.apache.flink.streaming.api.operators.async.queue.OrderedStreamElementQueue) OrderedStreamElementQueue(org.apache.flink.streaming.api.operators.async.queue.OrderedStreamElementQueue) CollectorOutput(org.apache.flink.streaming.util.CollectorOutput) Watermark(org.apache.flink.streaming.api.watermark.Watermark) StreamRecordQueueEntry(org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry) WatermarkQueueEntry(org.apache.flink.streaming.api.operators.async.queue.WatermarkQueueEntry) Test(org.junit.Test)

Aggregations

StreamRecordQueueEntry (org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry)3 ArrayList (java.util.ArrayList)2 OrderedStreamElementQueue (org.apache.flink.streaming.api.operators.async.queue.OrderedStreamElementQueue)2 StreamElementQueue (org.apache.flink.streaming.api.operators.async.queue.StreamElementQueue)2 WatermarkQueueEntry (org.apache.flink.streaming.api.operators.async.queue.WatermarkQueueEntry)2 Watermark (org.apache.flink.streaming.api.watermark.Watermark)2 StreamElement (org.apache.flink.streaming.runtime.streamrecord.StreamElement)2 StreamRecord (org.apache.flink.streaming.runtime.streamrecord.StreamRecord)2 CollectorOutput (org.apache.flink.streaming.util.CollectorOutput)2 Test (org.junit.Test)2 Collection (java.util.Collection)1 ExecutionException (java.util.concurrent.ExecutionException)1 TimeoutException (java.util.concurrent.TimeoutException)1 StreamElementQueueEntry (org.apache.flink.streaming.api.operators.async.queue.StreamElementQueueEntry)1 ProcessingTimeCallback (org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback)1