Examples with TestProcessingTimeService org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService used on opensource projects

Example 51 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class StreamSourceOperatorTest method testAutomaticWatermarkContext.

@Test
public void testAutomaticWatermarkContext() throws Exception {
    // regular stream source operator
    final StoppableStreamSource<String, InfiniteSource<String>> operator = new StoppableStreamSource<>(new InfiniteSource<String>());
    long watermarkInterval = 10;
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    processingTimeService.setCurrentTime(0);
    setupSourceOperator(operator, TimeCharacteristic.IngestionTime, watermarkInterval, 0, processingTimeService);
    final List<StreamElement> output = new ArrayList<>();
    StreamSourceContexts.getSourceContext(TimeCharacteristic.IngestionTime, operator.getContainingTask().getProcessingTimeService(), operator.getContainingTask().getCheckpointLock(), operator.getContainingTask().getStreamStatusMaintainer(), new CollectorOutput<String>(output), operator.getExecutionConfig().getAutoWatermarkInterval(), -1);
    for (long i = 1; i < 100; i += watermarkInterval) {
        processingTimeService.setCurrentTime(i);
    }
    assertTrue(output.size() == 9);
    long nextWatermark = 0;
    for (StreamElement el : output) {
        nextWatermark += watermarkInterval;
        Watermark wm = (Watermark) el;
        assertTrue(wm.getTimestamp() == nextWatermark);
    }
}

Example 52 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class AbstractFetcherTest method testIgnorePartitionStateSentinelInSnapshot.

@Test
public void testIgnorePartitionStateSentinelInSnapshot() throws Exception {
    final String testTopic = "test topic name";
    Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
    originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    originalPartitions.put(new KafkaTopicPartition(testTopic, 2), KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
    originalPartitions.put(new KafkaTopicPartition(testTopic, 3), KafkaTopicPartitionStateSentinel.EARLIEST_OFFSET);
    TestSourceContext<Long> sourceContext = new TestSourceContext<>();
    TestFetcher<Long> fetcher = new TestFetcher<>(sourceContext, originalPartitions, null, /* watermark strategy */
    new TestProcessingTimeService(), 0);
    synchronized (sourceContext.getCheckpointLock()) {
        HashMap<KafkaTopicPartition, Long> currentState = fetcher.snapshotCurrentState();
        fetcher.commitInternalOffsetsToKafka(currentState, new KafkaCommitCallback() {

            @Override
            public void onSuccess() {
            }

            @Override
            public void onException(Throwable cause) {
                throw new RuntimeException("Callback failed", cause);
            }
        });
        assertTrue(fetcher.getLastCommittedOffsets().isPresent());
        assertEquals(Collections.emptyMap(), fetcher.getLastCommittedOffsets().get());
    }
}

Example 53 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class AbstractFetcherTest method testSkipCorruptedRecord.

// ------------------------------------------------------------------------
// Record emitting tests
// ------------------------------------------------------------------------
@Test
public void testSkipCorruptedRecord() throws Exception {
    final String testTopic = "test topic name";
    Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
    originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    TestSourceContext<Long> sourceContext = new TestSourceContext<>();
    TestFetcher<Long> fetcher = new TestFetcher<>(sourceContext, originalPartitions, null, /* watermark strategy */
    new TestProcessingTimeService(), 0);
    final KafkaTopicPartitionState<Long, Object> partitionStateHolder = fetcher.subscribedPartitionStates().get(0);
    emitRecord(fetcher, 1L, partitionStateHolder, 1L);
    emitRecord(fetcher, 2L, partitionStateHolder, 2L);
    assertEquals(2L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(2L, partitionStateHolder.getOffset());
    // emit no records
    fetcher.emitRecordsWithTimestamps(emptyQueue(), partitionStateHolder, 3L, Long.MIN_VALUE);
    assertEquals(2L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(3L, // the offset in state still should have advanced
    partitionStateHolder.getOffset());
}

Example 54 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class TopicMetadataListenerTest method listenEmptyTopics.

@Test
void listenEmptyTopics() {
    TopicMetadataListener listener = new TopicMetadataListener();
    SinkConfiguration configuration = sinkConfiguration(Duration.ofMinutes(5).toMillis());
    TestProcessingTimeService timeService = new TestProcessingTimeService();
    List<String> topics = listener.availableTopics();
    assertThat(topics).isEmpty();
    listener.open(configuration, timeService);
    topics = listener.availableTopics();
    assertThat(topics).isEmpty();
}

Example 55 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class BatchExecutionInternalTimeServiceTest method testForEachProcessingTimeTimerUnsupported.

@Test
public void testForEachProcessingTimeTimerUnsupported() {
    expectedException.expect(UnsupportedOperationException.class);
    expectedException.expectMessage("The BatchExecutionInternalTimeService should not be used in State Processor API");
    BatchExecutionInternalTimeService<Object, Object> timeService = new BatchExecutionInternalTimeService<>(new TestProcessingTimeService(), LambdaTrigger.eventTimeTrigger(timer -> {
    }));
    timeService.forEachEventTimeTimer((o, aLong) -> fail("The forEachProcessingTimeTimer() should not be supported"));
}