Examples with Watermark org.apache.flink.streaming.api.watermark.Watermark used on opensource projects

Example 36 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class EmitterTest method testEmitterWithExceptions.

/**
	 * Tests that the emitter handles exceptions occurring in the {@link AsyncCollector} correctly.
	 */
@Test
public void testEmitterWithExceptions() throws Exception {
    Object lock = new Object();
    List<StreamElement> list = new ArrayList<>();
    Output<StreamRecord<Integer>> output = new CollectorOutput<>(list);
    List<StreamElement> expected = Arrays.asList(new StreamRecord<>(1, 0L), new Watermark(3L));
    OperatorActions operatorActions = mock(OperatorActions.class);
    final int capacity = 3;
    StreamElementQueue queue = new OrderedStreamElementQueue(capacity, executor, operatorActions);
    final Emitter<Integer> emitter = new Emitter<>(lock, output, queue, operatorActions);
    final Thread emitterThread = new Thread(emitter);
    emitterThread.start();
    final Exception testException = new Exception("Test exception");
    try {
        StreamRecordQueueEntry<Integer> record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L));
        StreamRecordQueueEntry<Integer> record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L));
        WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(3L));
        queue.put(record1);
        queue.put(record2);
        queue.put(watermark1);
        record2.collect(testException);
        record1.collect(Arrays.asList(1));
        synchronized (lock) {
            while (!queue.isEmpty()) {
                lock.wait();
            }
        }
        Assert.assertEquals(expected, list);
        ArgumentCaptor<Throwable> argumentCaptor = ArgumentCaptor.forClass(Throwable.class);
        verify(operatorActions).failOperator(argumentCaptor.capture());
        Throwable failureCause = argumentCaptor.getValue();
        Assert.assertNotNull(failureCause.getCause());
        Assert.assertTrue(failureCause.getCause() instanceof ExecutionException);
        Assert.assertNotNull(failureCause.getCause().getCause());
        Assert.assertEquals(testException, failureCause.getCause().getCause());
    } finally {
        emitter.stop();
        emitterThread.interrupt();
    }
}

Example 37 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class StreamElementQueueTest method testPoll.

@Test
public void testPoll() throws InterruptedException {
    OperatorActions operatorActions = mock(OperatorActions.class);
    StreamElementQueue queue = createStreamElementQueue(2, operatorActions);
    WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(new Watermark(0L));
    StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(42, 1L));
    queue.put(watermarkQueueEntry);
    queue.put(streamRecordQueueEntry);
    Assert.assertEquals(watermarkQueueEntry, queue.peekBlockingly());
    Assert.assertEquals(2, queue.size());
    Assert.assertEquals(watermarkQueueEntry, queue.poll());
    Assert.assertEquals(1, queue.size());
    streamRecordQueueEntry.collect(Collections.<Integer>emptyList());
    Assert.assertEquals(streamRecordQueueEntry, queue.poll());
    Assert.assertEquals(0, queue.size());
    Assert.assertTrue(queue.isEmpty());
    verify(operatorActions, never()).failOperator(any(Exception.class));
}

Example 38 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class StreamElementQueueTest method testPut.

@Test
public void testPut() throws InterruptedException {
    OperatorActions operatorActions = mock(OperatorActions.class);
    StreamElementQueue queue = createStreamElementQueue(2, operatorActions);
    final Watermark watermark = new Watermark(0L);
    final StreamRecord<Integer> streamRecord = new StreamRecord<>(42, 1L);
    final Watermark nextWatermark = new Watermark(2L);
    final WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(watermark);
    final StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(streamRecord);
    queue.put(watermarkQueueEntry);
    queue.put(streamRecordQueueEntry);
    Assert.assertEquals(2, queue.size());
    Assert.assertFalse(queue.tryPut(new WatermarkQueueEntry(nextWatermark)));
    Collection<StreamElementQueueEntry<?>> actualValues = queue.values();
    List<StreamElementQueueEntry<?>> expectedValues = Arrays.asList(watermarkQueueEntry, streamRecordQueueEntry);
    Assert.assertEquals(expectedValues, actualValues);
    verify(operatorActions, never()).failOperator(any(Exception.class));
}

Example 39 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class StreamElementQueueTest method testBlockingPoll.

/**
	 * Test that a poll operation on an empty queue blocks.
	 */
@Test
public void testBlockingPoll() throws Exception {
    OperatorActions operatorActions = mock(OperatorActions.class);
    final StreamElementQueue queue = createStreamElementQueue(1, operatorActions);
    WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(new Watermark(1L));
    StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 2L));
    Assert.assertTrue(queue.isEmpty());
    Future<AsyncResult> peekOperation = FlinkFuture.supplyAsync(new Callable<AsyncResult>() {

        @Override
        public AsyncResult call() throws Exception {
            return queue.peekBlockingly();
        }
    }, executor);
    Thread.sleep(10L);
    Assert.assertFalse(peekOperation.isDone());
    queue.put(watermarkQueueEntry);
    AsyncResult watermarkResult = peekOperation.get();
    Assert.assertEquals(watermarkQueueEntry, watermarkResult);
    Assert.assertEquals(1, queue.size());
    Assert.assertEquals(watermarkQueueEntry, queue.poll());
    Assert.assertTrue(queue.isEmpty());
    Future<AsyncResult> pollOperation = FlinkFuture.supplyAsync(new Callable<AsyncResult>() {

        @Override
        public AsyncResult call() throws Exception {
            return queue.poll();
        }
    }, executor);
    Thread.sleep(10L);
    Assert.assertFalse(pollOperation.isDone());
    queue.put(streamRecordQueueEntry);
    Thread.sleep(10L);
    Assert.assertFalse(pollOperation.isDone());
    streamRecordQueueEntry.collect(Collections.<Integer>emptyList());
    Assert.assertEquals(streamRecordQueueEntry, pollOperation.get());
    Assert.assertTrue(queue.isEmpty());
    verify(operatorActions, never()).failOperator(any(Exception.class));
}

Example 40 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class UnorderedStreamElementQueueTest method testCompletionOrder.

/**
	 * Tests that only elements before the oldest watermark are returned if they are completed.
	 */
@Test
public void testCompletionOrder() throws Exception {
    OperatorActions operatorActions = mock(OperatorActions.class);
    final UnorderedStreamElementQueue queue = new UnorderedStreamElementQueue(8, executor, operatorActions);
    StreamRecordQueueEntry<Integer> record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L));
    StreamRecordQueueEntry<Integer> record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L));
    WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(2L));
    StreamRecordQueueEntry<Integer> record3 = new StreamRecordQueueEntry<>(new StreamRecord<>(3, 3L));
    StreamRecordQueueEntry<Integer> record4 = new StreamRecordQueueEntry<>(new StreamRecord<>(4, 4L));
    WatermarkQueueEntry watermark2 = new WatermarkQueueEntry(new Watermark(5L));
    StreamRecordQueueEntry<Integer> record5 = new StreamRecordQueueEntry<>(new StreamRecord<>(5, 6L));
    StreamRecordQueueEntry<Integer> record6 = new StreamRecordQueueEntry<>(new StreamRecord<>(6, 7L));
    List<StreamElementQueueEntry<?>> entries = Arrays.asList(record1, record2, watermark1, record3, record4, watermark2, record5, record6);
    // The queue should look like R1, R2, W1, R3, R4, W2, R5, R6
    for (StreamElementQueueEntry<?> entry : entries) {
        queue.put(entry);
    }
    Assert.assertTrue(8 == queue.size());
    Future<AsyncResult> firstPoll = FlinkFuture.supplyAsync(new Callable<AsyncResult>() {

        @Override
        public AsyncResult call() throws Exception {
            return queue.poll();
        }
    }, executor);
    // this should not fulfill the poll, because R3 is behind W1
    record3.collect(Collections.<Integer>emptyList());
    Thread.sleep(10L);
    Assert.assertFalse(firstPoll.isDone());
    record2.collect(Collections.<Integer>emptyList());
    Assert.assertEquals(record2, firstPoll.get());
    Future<AsyncResult> secondPoll = FlinkFuture.supplyAsync(new Callable<AsyncResult>() {

        @Override
        public AsyncResult call() throws Exception {
            return queue.poll();
        }
    }, executor);
    record6.collect(Collections.<Integer>emptyList());
    record4.collect(Collections.<Integer>emptyList());
    Thread.sleep(10L);
    // The future should not be completed because R1 has not been completed yet
    Assert.assertFalse(secondPoll.isDone());
    record1.collect(Collections.<Integer>emptyList());
    Assert.assertEquals(record1, secondPoll.get());
    // Now W1, R3, R4 and W2 are completed and should be pollable
    Assert.assertEquals(watermark1, queue.poll());
    // The order of R3 and R4 is not specified
    Set<AsyncResult> expected = new HashSet<>(2);
    expected.add(record3);
    expected.add(record4);
    Set<AsyncResult> actual = new HashSet<>(2);
    actual.add(queue.poll());
    actual.add(queue.poll());
    Assert.assertEquals(expected, actual);
    Assert.assertEquals(watermark2, queue.poll());
    // since R6 has been completed before and W2 has been consumed, we should be able to poll
    // this record as well
    Assert.assertEquals(record6, queue.poll());
    // only R5 left in the queue
    Assert.assertTrue(1 == queue.size());
    Future<AsyncResult> thirdPoll = FlinkFuture.supplyAsync(new Callable<AsyncResult>() {

        @Override
        public AsyncResult call() throws Exception {
            return queue.poll();
        }
    }, executor);
    Thread.sleep(10L);
    Assert.assertFalse(thirdPoll.isDone());
    record5.collect(Collections.<Integer>emptyList());
    Assert.assertEquals(record5, thirdPoll.get());
    Assert.assertTrue(queue.isEmpty());
    verify(operatorActions, never()).failOperator(any(Exception.class));
}