Examples with TestProcessingTimeService org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService used on opensource projects

Example 11 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class HeapInternalTimerServiceTest method testTimerAssignmentToKeyGroups.

@Test
public void testTimerAssignmentToKeyGroups() {
    int totalNoOfTimers = 100;
    int totalNoOfKeyGroups = 100;
    int startKeyGroupIdx = 0;
    // we have 0 to 99
    int endKeyGroupIdx = totalNoOfKeyGroups - 1;
    @SuppressWarnings("unchecked") Set<InternalTimer<Integer, String>>[] expectedNonEmptyTimerSets = new HashSet[totalNoOfKeyGroups];
    TestKeyContext keyContext = new TestKeyContext();
    HeapInternalTimerService<Integer, String> timerService = new HeapInternalTimerService<>(totalNoOfKeyGroups, new KeyGroupRange(startKeyGroupIdx, endKeyGroupIdx), keyContext, new TestProcessingTimeService());
    timerService.startTimerService(IntSerializer.INSTANCE, StringSerializer.INSTANCE, mock(Triggerable.class));
    for (int i = 0; i < totalNoOfTimers; i++) {
        // create the timer to be registered
        InternalTimer<Integer, String> timer = new InternalTimer<>(10 + i, i, "hello_world_" + i);
        int keyGroupIdx = KeyGroupRangeAssignment.assignToKeyGroup(timer.getKey(), totalNoOfKeyGroups);
        // add it in the adequate expected set of timers per keygroup
        Set<InternalTimer<Integer, String>> timerSet = expectedNonEmptyTimerSets[keyGroupIdx];
        if (timerSet == null) {
            timerSet = new HashSet<>();
            expectedNonEmptyTimerSets[keyGroupIdx] = timerSet;
        }
        timerSet.add(timer);
        // register the timer as both processing and event time one
        keyContext.setCurrentKey(timer.getKey());
        timerService.registerEventTimeTimer(timer.getNamespace(), timer.getTimestamp());
        timerService.registerProcessingTimeTimer(timer.getNamespace(), timer.getTimestamp());
    }
    Set<InternalTimer<Integer, String>>[] eventTimeTimers = timerService.getEventTimeTimersPerKeyGroup();
    Set<InternalTimer<Integer, String>>[] processingTimeTimers = timerService.getProcessingTimeTimersPerKeyGroup();
    // finally verify that the actual timers per key group sets are the expected ones.
    for (int i = 0; i < expectedNonEmptyTimerSets.length; i++) {
        Set<InternalTimer<Integer, String>> expected = expectedNonEmptyTimerSets[i];
        Set<InternalTimer<Integer, String>> actualEvent = eventTimeTimers[i];
        Set<InternalTimer<Integer, String>> actualProcessing = processingTimeTimers[i];
        if (expected == null) {
            Assert.assertNull(actualEvent);
            Assert.assertNull(actualProcessing);
        } else {
            Assert.assertArrayEquals(expected.toArray(), actualEvent.toArray());
            Assert.assertArrayEquals(expected.toArray(), actualProcessing.toArray());
        }
    }
}

Example 12 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class HeapInternalTimerServiceTest method testSetAndFireEventTimeTimers.

/**
	 * This also verifies that we don't have leakage between keys/namespaces.
	 */
@Test
public void testSetAndFireEventTimeTimers() throws Exception {
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    HeapInternalTimerService<Integer, String> timerService = createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
    // get two different keys
    int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    int key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    while (key2 == key1) {
        key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    }
    keyContext.setCurrentKey(key1);
    timerService.registerEventTimeTimer("ciao", 10);
    timerService.registerEventTimeTimer("hello", 10);
    keyContext.setCurrentKey(key2);
    timerService.registerEventTimeTimer("ciao", 10);
    timerService.registerEventTimeTimer("hello", 10);
    assertEquals(4, timerService.numEventTimeTimers());
    assertEquals(2, timerService.numEventTimeTimers("hello"));
    assertEquals(2, timerService.numEventTimeTimers("ciao"));
    timerService.advanceWatermark(10);
    verify(mockTriggerable, times(4)).onEventTime(anyInternalTimer());
    verify(mockTriggerable, times(1)).onEventTime(eq(new InternalTimer<>(10, key1, "ciao")));
    verify(mockTriggerable, times(1)).onEventTime(eq(new InternalTimer<>(10, key1, "hello")));
    verify(mockTriggerable, times(1)).onEventTime(eq(new InternalTimer<>(10, key2, "ciao")));
    verify(mockTriggerable, times(1)).onEventTime(eq(new InternalTimer<>(10, key2, "hello")));
    assertEquals(0, timerService.numEventTimeTimers());
}

Example 13 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class StreamSourceOperatorTest method testLatencyMarkEmission.

/**
	 * Test that latency marks are emitted
	 */
@Test
public void testLatencyMarkEmission() throws Exception {
    final List<StreamElement> output = new ArrayList<>();
    final long maxProcessingTime = 100L;
    final long latencyMarkInterval = 10L;
    final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
    testProcessingTimeService.setCurrentTime(0L);
    final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);
    // regular stream source operator
    final StreamSource<Long, ProcessingTimeServiceSource> operator = new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes));
    // emit latency marks every 10 milliseconds.
    setupSourceOperator(operator, TimeCharacteristic.EventTime, 0, latencyMarkInterval, testProcessingTimeService);
    // run and wait to be stopped
    operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<Long>(output));
    int numberLatencyMarkers = (int) (maxProcessingTime / latencyMarkInterval) + 1;
    assertEquals(// + 1 is the final watermark element
    numberLatencyMarkers + 1, output.size());
    long timestamp = 0L;
    int i = 0;
    // and that its only latency markers + a final watermark
    for (; i < output.size() - 1; i++) {
        StreamElement se = output.get(i);
        Assert.assertTrue(se.isLatencyMarker());
        Assert.assertEquals(-1, se.asLatencyMarker().getVertexID());
        Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex());
        Assert.assertTrue(se.asLatencyMarker().getMarkedTime() == timestamp);
        timestamp += latencyMarkInterval;
    }
    Assert.assertTrue(output.get(i).isWatermark());
}

Example 14 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class AbstractFetcherTest method testConcurrentPartitionsDiscoveryAndLoopFetching.

@Test
public void testConcurrentPartitionsDiscoveryAndLoopFetching() throws Exception {
    // test data
    final KafkaTopicPartition testPartition = new KafkaTopicPartition("test", 42);
    // ----- create the test fetcher -----
    SourceContext<String> sourceContext = new TestSourceContext<>();
    Map<KafkaTopicPartition, Long> partitionsWithInitialOffsets = Collections.singletonMap(testPartition, KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
    final OneShotLatch fetchLoopWaitLatch = new OneShotLatch();
    final OneShotLatch stateIterationBlockLatch = new OneShotLatch();
    final TestFetcher<String> fetcher = new TestFetcher<>(sourceContext, partitionsWithInitialOffsets, null, /* watermark strategy */
    new TestProcessingTimeService(), 10, fetchLoopWaitLatch, stateIterationBlockLatch);
    // ----- run the fetcher -----
    final CheckedThread checkedThread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            fetcher.runFetchLoop();
        }
    };
    checkedThread.start();
    // wait until state iteration begins before adding discovered partitions
    fetchLoopWaitLatch.await();
    fetcher.addDiscoveredPartitions(Collections.singletonList(testPartition));
    stateIterationBlockLatch.trigger();
    checkedThread.sync();
}

Example 15 with TestProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService in project flink by apache.

the class TopicMetadataListenerTest method fetchTopicPartitionUpdate.

@Test
void fetchTopicPartitionUpdate() throws Exception {
    String topic = randomAlphabetic(10);
    operator().createTopic(topic, 8);
    long interval = Duration.ofMinutes(20).toMillis();
    TopicMetadataListener listener = new TopicMetadataListener(singletonList(topic));
    SinkConfiguration configuration = sinkConfiguration(interval);
    TestProcessingTimeService timeService = new TestProcessingTimeService();
    timeService.setCurrentTime(System.currentTimeMillis());
    listener.open(configuration, timeService);
    List<String> topics = listener.availableTopics();
    List<String> desiredTopics = topicPartitions(topic, 8);
    assertThat(topics).isEqualTo(desiredTopics);
    // Increase topic partitions and trigger the metadata update logic.
    operator().increaseTopicPartitions(topic, 16);
    timeService.advance(interval);
    topics = listener.availableTopics();
    desiredTopics = topicPartitions(topic, 16);
    assertThat(topics).isEqualTo(desiredTopics);
}