Examples with OperatorActions org.apache.flink.streaming.api.operators.async.OperatorActions used on opensource projects

Example 1 with OperatorActions

use of org.apache.flink.streaming.api.operators.async.OperatorActions in project flink by apache.

the class StreamElementQueueTest method testPoll.

@Test
public void testPoll() throws InterruptedException {
    OperatorActions operatorActions = mock(OperatorActions.class);
    StreamElementQueue queue = createStreamElementQueue(2, operatorActions);
    WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(new Watermark(0L));
    StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(42, 1L));
    queue.put(watermarkQueueEntry);
    queue.put(streamRecordQueueEntry);
    Assert.assertEquals(watermarkQueueEntry, queue.peekBlockingly());
    Assert.assertEquals(2, queue.size());
    Assert.assertEquals(watermarkQueueEntry, queue.poll());
    Assert.assertEquals(1, queue.size());
    streamRecordQueueEntry.collect(Collections.<Integer>emptyList());
    Assert.assertEquals(streamRecordQueueEntry, queue.poll());
    Assert.assertEquals(0, queue.size());
    Assert.assertTrue(queue.isEmpty());
    verify(operatorActions, never()).failOperator(any(Exception.class));
}

Example 2 with OperatorActions

use of org.apache.flink.streaming.api.operators.async.OperatorActions in project flink by apache.

the class StreamElementQueueTest method testBlockingPut.

/**
	 * Tests that a put operation blocks if the queue is full.
	 */
@Test
public void testBlockingPut() throws Exception {
    OperatorActions operatorActions = mock(OperatorActions.class);
    final StreamElementQueue queue = createStreamElementQueue(1, operatorActions);
    StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(42, 0L));
    final StreamRecordQueueEntry<Integer> streamRecordQueueEntry2 = new StreamRecordQueueEntry<>(new StreamRecord<>(43, 1L));
    queue.put(streamRecordQueueEntry);
    Assert.assertEquals(1, queue.size());
    Future<Void> putOperation = FlinkFuture.supplyAsync(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            queue.put(streamRecordQueueEntry2);
            return null;
        }
    }, executor);
    // give the future a chance to complete
    Thread.sleep(10L);
    // but it shouldn't ;-)
    Assert.assertFalse(putOperation.isDone());
    streamRecordQueueEntry.collect(Collections.<Integer>emptyList());
    // polling the completed head element frees the queue again
    Assert.assertEquals(streamRecordQueueEntry, queue.poll());
    // now the put operation should complete
    putOperation.get();
    verify(operatorActions, never()).failOperator(any(Exception.class));
}

Example 3 with OperatorActions

use of org.apache.flink.streaming.api.operators.async.OperatorActions in project flink by apache.

the class StreamElementQueueTest method testPut.

@Test
public void testPut() throws InterruptedException {
    OperatorActions operatorActions = mock(OperatorActions.class);
    StreamElementQueue queue = createStreamElementQueue(2, operatorActions);
    final Watermark watermark = new Watermark(0L);
    final StreamRecord<Integer> streamRecord = new StreamRecord<>(42, 1L);
    final Watermark nextWatermark = new Watermark(2L);
    final WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(watermark);
    final StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(streamRecord);
    queue.put(watermarkQueueEntry);
    queue.put(streamRecordQueueEntry);
    Assert.assertEquals(2, queue.size());
    Assert.assertFalse(queue.tryPut(new WatermarkQueueEntry(nextWatermark)));
    Collection<StreamElementQueueEntry<?>> actualValues = queue.values();
    List<StreamElementQueueEntry<?>> expectedValues = Arrays.asList(watermarkQueueEntry, streamRecordQueueEntry);
    Assert.assertEquals(expectedValues, actualValues);
    verify(operatorActions, never()).failOperator(any(Exception.class));
}

Example 4 with OperatorActions

use of org.apache.flink.streaming.api.operators.async.OperatorActions in project flink by apache.

the class StreamElementQueueTest method testBlockingPoll.

/**
	 * Test that a poll operation on an empty queue blocks.
	 */
@Test
public void testBlockingPoll() throws Exception {
    OperatorActions operatorActions = mock(OperatorActions.class);
    final StreamElementQueue queue = createStreamElementQueue(1, operatorActions);
    WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(new Watermark(1L));
    StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 2L));
    Assert.assertTrue(queue.isEmpty());
    Future<AsyncResult> peekOperation = FlinkFuture.supplyAsync(new Callable<AsyncResult>() {

        @Override
        public AsyncResult call() throws Exception {
            return queue.peekBlockingly();
        }
    }, executor);
    Thread.sleep(10L);
    Assert.assertFalse(peekOperation.isDone());
    queue.put(watermarkQueueEntry);
    AsyncResult watermarkResult = peekOperation.get();
    Assert.assertEquals(watermarkQueueEntry, watermarkResult);
    Assert.assertEquals(1, queue.size());
    Assert.assertEquals(watermarkQueueEntry, queue.poll());
    Assert.assertTrue(queue.isEmpty());
    Future<AsyncResult> pollOperation = FlinkFuture.supplyAsync(new Callable<AsyncResult>() {

        @Override
        public AsyncResult call() throws Exception {
            return queue.poll();
        }
    }, executor);
    Thread.sleep(10L);
    Assert.assertFalse(pollOperation.isDone());
    queue.put(streamRecordQueueEntry);
    Thread.sleep(10L);
    Assert.assertFalse(pollOperation.isDone());
    streamRecordQueueEntry.collect(Collections.<Integer>emptyList());
    Assert.assertEquals(streamRecordQueueEntry, pollOperation.get());
    Assert.assertTrue(queue.isEmpty());
    verify(operatorActions, never()).failOperator(any(Exception.class));
}

Example 5 with OperatorActions

use of org.apache.flink.streaming.api.operators.async.OperatorActions in project flink by apache.

the class UnorderedStreamElementQueueTest method testCompletionOrder.

/**
	 * Tests that only elements before the oldest watermark are returned if they are completed.
	 */
@Test
public void testCompletionOrder() throws Exception {
    OperatorActions operatorActions = mock(OperatorActions.class);
    final UnorderedStreamElementQueue queue = new UnorderedStreamElementQueue(8, executor, operatorActions);
    StreamRecordQueueEntry<Integer> record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L));
    StreamRecordQueueEntry<Integer> record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L));
    WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(2L));
    StreamRecordQueueEntry<Integer> record3 = new StreamRecordQueueEntry<>(new StreamRecord<>(3, 3L));
    StreamRecordQueueEntry<Integer> record4 = new StreamRecordQueueEntry<>(new StreamRecord<>(4, 4L));
    WatermarkQueueEntry watermark2 = new WatermarkQueueEntry(new Watermark(5L));
    StreamRecordQueueEntry<Integer> record5 = new StreamRecordQueueEntry<>(new StreamRecord<>(5, 6L));
    StreamRecordQueueEntry<Integer> record6 = new StreamRecordQueueEntry<>(new StreamRecord<>(6, 7L));
    List<StreamElementQueueEntry<?>> entries = Arrays.asList(record1, record2, watermark1, record3, record4, watermark2, record5, record6);
    // The queue should look like R1, R2, W1, R3, R4, W2, R5, R6
    for (StreamElementQueueEntry<?> entry : entries) {
        queue.put(entry);
    }
    Assert.assertTrue(8 == queue.size());
    Future<AsyncResult> firstPoll = FlinkFuture.supplyAsync(new Callable<AsyncResult>() {

        @Override
        public AsyncResult call() throws Exception {
            return queue.poll();
        }
    }, executor);
    // this should not fulfill the poll, because R3 is behind W1
    record3.collect(Collections.<Integer>emptyList());
    Thread.sleep(10L);
    Assert.assertFalse(firstPoll.isDone());
    record2.collect(Collections.<Integer>emptyList());
    Assert.assertEquals(record2, firstPoll.get());
    Future<AsyncResult> secondPoll = FlinkFuture.supplyAsync(new Callable<AsyncResult>() {

        @Override
        public AsyncResult call() throws Exception {
            return queue.poll();
        }
    }, executor);
    record6.collect(Collections.<Integer>emptyList());
    record4.collect(Collections.<Integer>emptyList());
    Thread.sleep(10L);
    // The future should not be completed because R1 has not been completed yet
    Assert.assertFalse(secondPoll.isDone());
    record1.collect(Collections.<Integer>emptyList());
    Assert.assertEquals(record1, secondPoll.get());
    // Now W1, R3, R4 and W2 are completed and should be pollable
    Assert.assertEquals(watermark1, queue.poll());
    // The order of R3 and R4 is not specified
    Set<AsyncResult> expected = new HashSet<>(2);
    expected.add(record3);
    expected.add(record4);
    Set<AsyncResult> actual = new HashSet<>(2);
    actual.add(queue.poll());
    actual.add(queue.poll());
    Assert.assertEquals(expected, actual);
    Assert.assertEquals(watermark2, queue.poll());
    // since R6 has been completed before and W2 has been consumed, we should be able to poll
    // this record as well
    Assert.assertEquals(record6, queue.poll());
    // only R5 left in the queue
    Assert.assertTrue(1 == queue.size());
    Future<AsyncResult> thirdPoll = FlinkFuture.supplyAsync(new Callable<AsyncResult>() {

        @Override
        public AsyncResult call() throws Exception {
            return queue.poll();
        }
    }, executor);
    Thread.sleep(10L);
    Assert.assertFalse(thirdPoll.isDone());
    record5.collect(Collections.<Integer>emptyList());
    Assert.assertEquals(record5, thirdPoll.get());
    Assert.assertTrue(queue.isEmpty());
    verify(operatorActions, never()).failOperator(any(Exception.class));
}