Examples with ManuallyTriggeredScheduledExecutorService org.apache.flink.core.testutils.ManuallyTriggeredScheduledExecutorService used on opensource projects

Example 1 with ManuallyTriggeredScheduledExecutorService

use of org.apache.flink.core.testutils.ManuallyTriggeredScheduledExecutorService in project flink by apache.

the class BatchingStateChangeUploaderTest method testDelay.

@Test
public void testDelay() throws Exception {
    int delayMs = 50;
    ManuallyTriggeredScheduledExecutorService scheduler = new ManuallyTriggeredScheduledExecutorService();
    withStore(delayMs, MAX_BYTES_IN_FLIGHT, MAX_BYTES_IN_FLIGHT, scheduler, (store, probe) -> {
        scheduler.triggerAll();
        List<StateChangeSet> changeSets = getChanges(4);
        upload(store, changeSets);
        assertTrue(probe.getUploaded().isEmpty());
        assertTrue(scheduler.getAllNonPeriodicScheduledTask().stream().anyMatch(scheduled -> scheduled.getDelay(MILLISECONDS) == delayMs));
        scheduler.triggerAllNonPeriodicTasks();
        assertEquals(changeSets, probe.getUploaded());
    });
}

Example 2 with ManuallyTriggeredScheduledExecutorService

use of org.apache.flink.core.testutils.ManuallyTriggeredScheduledExecutorService in project flink by apache.

the class ChangelogStorageMetricsTest method testQueueSize.

@Test
public void testQueueSize() throws Exception {
    AtomicReference<Gauge<Integer>> queueSizeGauge = new AtomicReference<>();
    ChangelogStorageMetricGroup metrics = new ChangelogStorageMetricGroup(new TaskManagerJobMetricGroup(TestingMetricRegistry.builder().setRegisterConsumer((metric, name, unused) -> {
        if (name.equals(CHANGELOG_STORAGE_UPLOAD_QUEUE_SIZE)) {
            queueSizeGauge.set((Gauge<Integer>) metric);
        }
    }).build(), createUnregisteredTaskManagerMetricGroup(), new JobID(), "test"));
    Path path = Path.fromLocalFile(temporaryFolder.newFolder());
    StateChangeFsUploader delegate = new StateChangeFsUploader(path, path.getFileSystem(), false, 100, metrics);
    ManuallyTriggeredScheduledExecutorService scheduler = new ManuallyTriggeredScheduledExecutorService();
    BatchingStateChangeUploader batcher = new BatchingStateChangeUploader(Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, RetryPolicy.NONE, delegate, scheduler, new RetryingExecutor(1, metrics.getAttemptsPerUpload()), metrics);
    try (FsStateChangelogStorage storage = new FsStateChangelogStorage(batcher, Long.MAX_VALUE)) {
        FsStateChangelogWriter writer = storage.createWriter("writer", EMPTY_KEY_GROUP_RANGE);
        int numUploads = 11;
        for (int i = 0; i < numUploads; i++) {
            SequenceNumber from = writer.nextSequenceNumber();
            writer.append(0, new byte[] { 0 });
            writer.persist(from);
        }
        assertEquals(numUploads, (int) queueSizeGauge.get().getValue());
        scheduler.triggerScheduledTasks();
        assertEquals(0, (int) queueSizeGauge.get().getValue());
    }
}

Example 3 with ManuallyTriggeredScheduledExecutorService

use of org.apache.flink.core.testutils.ManuallyTriggeredScheduledExecutorService in project flink by apache.

the class DefaultTimerServiceTest method testUnregisterAllTimeouts.

/**
 * Tests that all registered timeouts can be unregistered.
 */
@Test
public void testUnregisterAllTimeouts() throws Exception {
    final ManuallyTriggeredScheduledExecutorService scheduledExecutorService = new ManuallyTriggeredScheduledExecutorService();
    DefaultTimerService<AllocationID> timerService = new DefaultTimerService<>(scheduledExecutorService, 100L);
    timerService.start((ignoredA, ignoredB) -> {
    });
    timerService.registerTimeout(new AllocationID(), 10, TimeUnit.SECONDS);
    timerService.registerTimeout(new AllocationID(), 10, TimeUnit.SECONDS);
    timerService.unregisterAllTimeouts();
    Map<?, ?> timeouts = timerService.getTimeouts();
    assertTrue(timeouts.isEmpty());
    for (ScheduledFuture<?> scheduledTask : scheduledExecutorService.getAllScheduledTasks()) {
        assertThat(scheduledTask.isCancelled(), is(true));
    }
}

Example 4 with ManuallyTriggeredScheduledExecutorService

use of org.apache.flink.core.testutils.ManuallyTriggeredScheduledExecutorService in project flink by apache.

the class ChangelogStorageMetricsTest method testUploadBatchSizes.

@Test
public void testUploadBatchSizes() throws Exception {
    int numWriters = 5, numUploads = 5;
    ChangelogStorageMetricGroup metrics = new ChangelogStorageMetricGroup(createUnregisteredTaskManagerJobMetricGroup());
    Path basePath = Path.fromLocalFile(temporaryFolder.newFolder());
    StateChangeFsUploader uploader = new StateChangeFsUploader(basePath, basePath.getFileSystem(), false, 100, metrics);
    ManuallyTriggeredScheduledExecutorService scheduler = new ManuallyTriggeredScheduledExecutorService();
    BatchingStateChangeUploader batcher = new BatchingStateChangeUploader(Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, RetryPolicy.NONE, uploader, scheduler, new RetryingExecutor(1, metrics.getAttemptsPerUpload()), metrics);
    FsStateChangelogStorage storage = new FsStateChangelogStorage(batcher, Integer.MAX_VALUE);
    FsStateChangelogWriter[] writers = new FsStateChangelogWriter[numWriters];
    for (int i = 0; i < numWriters; i++) {
        writers[i] = storage.createWriter(Integer.toString(i), EMPTY_KEY_GROUP_RANGE);
    }
    try {
        for (int upload = 0; upload < numUploads; upload++) {
            for (int writer = 0; writer < numWriters; writer++) {
                // with all thresholds on MAX and manually triggered executor, this shouldn't
                // cause actual uploads
                SequenceNumber from = writers[writer].nextSequenceNumber();
                writers[writer].append(0, new byte[] { 0, 1, 2, 3 });
                writers[writer].persist(from);
            }
            // now the uploads should be grouped and executed at once
            scheduler.triggerScheduledTasks();
        }
        assertEquals(numWriters, metrics.getUploadBatchSizes().getStatistics().getMin());
        assertEquals(numWriters, metrics.getUploadBatchSizes().getStatistics().getMax());
    } finally {
        storage.close();
    }
}

Example 5 with ManuallyTriggeredScheduledExecutorService

use of org.apache.flink.core.testutils.ManuallyTriggeredScheduledExecutorService in project flink by apache.

the class DefaultTimerServiceTest method createAndStartTimerService.

private static DefaultTimerService<AllocationID> createAndStartTimerService() {
    final ManuallyTriggeredScheduledExecutorService scheduledExecutorService = new ManuallyTriggeredScheduledExecutorService();
    DefaultTimerService<AllocationID> timerService = new DefaultTimerService<>(scheduledExecutorService, 100L);
    timerService.start((ignoredA, ignoredB) -> {
    });
    return timerService;
}