Examples with UploadTask org.apache.flink.changelog.fs.StateChangeUploadScheduler.UploadTask used on opensource projects

Example 6 with UploadTask

use of org.apache.flink.changelog.fs.StateChangeUploadScheduler.UploadTask in project flink by splunk.

the class FsStateChangelogWriter method persistInternal.

private CompletableFuture<ChangelogStateHandleStreamImpl> persistInternal(SequenceNumber from) throws IOException {
    synchronized (lock) {
        ensureCanPersist(from);
        rollover();
        Map<SequenceNumber, StateChangeSet> toUpload = drainTailMap(notUploaded, from);
        NavigableMap<SequenceNumber, UploadResult> readyToReturn = uploaded.tailMap(from, true);
        LOG.debug("collected readyToReturn: {}, toUpload: {}", readyToReturn, toUpload);
        SequenceNumberRange range = SequenceNumberRange.generic(from, activeSequenceNumber);
        if (range.size() == readyToReturn.size()) {
            checkState(toUpload.isEmpty());
            return completedFuture(buildHandle(keyGroupRange, readyToReturn, 0L));
        } else {
            CompletableFuture<ChangelogStateHandleStreamImpl> future = new CompletableFuture<>();
            uploadCompletionListeners.add(new UploadCompletionListener(keyGroupRange, range, readyToReturn, future));
            if (!toUpload.isEmpty()) {
                uploader.upload(new UploadTask(toUpload.values(), this::handleUploadSuccess, this::handleUploadFailure));
            }
            return future;
        }
    }
}

Example 7 with UploadTask

use of org.apache.flink.changelog.fs.StateChangeUploadScheduler.UploadTask in project flink by splunk.

the class BatchingStateChangeUploadSchedulerTest method testUploadTimeout.

@Test
public void testUploadTimeout() throws Exception {
    AtomicReference<List<SequenceNumber>> failed = new AtomicReference<>();
    UploadTask upload = new UploadTask(getChanges(4), unused -> {
    }, (sqn, error) -> failed.set(sqn));
    ManuallyTriggeredScheduledExecutorService executorService = new ManuallyTriggeredScheduledExecutorService();
    try (BatchingStateChangeUploadScheduler store = scheduler(1, executorService, new BlockingUploader(), 1)) {
        store.upload(upload);
        Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5));
        while (!upload.finished.get() && deadline.hasTimeLeft()) {
            executorService.triggerScheduledTasks();
            executorService.triggerAll();
            Thread.sleep(10);
        }
    }
    assertTrue(upload.finished.get());
    assertEquals(upload.changeSets.stream().map(StateChangeSet::getSequenceNumber).collect(Collectors.toSet()), new HashSet<>(failed.get()));
}