Examples with UploadTask org.apache.flink.changelog.fs.StateChangeUploader.UploadTask used on opensource projects

Example 1 with UploadTask

use of org.apache.flink.changelog.fs.StateChangeUploader.UploadTask in project flink by apache.

the class BatchingStateChangeUploaderTest method testRetry.

/**
 * Test integration with {@link RetryingExecutor}.
 */
@Test
public void testRetry() throws Exception {
    final int maxAttempts = 5;
    try (BatchingStateChangeUploader store = new BatchingStateChangeUploader(0, 0, MAX_BYTES_IN_FLIGHT, RetryPolicy.fixed(maxAttempts, 0, 0), new TestingStateChangeUploader() {

        final AtomicInteger currentAttempt = new AtomicInteger(0);

        @Override
        public void upload(UploadTask uploadTask) throws IOException {
            if (currentAttempt.getAndIncrement() < maxAttempts - 1) {
                throw new IOException();
            } else {
                uploadTask.complete(emptyList());
            }
        }
    }, new DirectScheduledExecutorService(), new RetryingExecutor(new DirectScheduledExecutorService(), createUnregisteredChangelogStorageMetricGroup().getAttemptsPerUpload()), createUnregisteredChangelogStorageMetricGroup())) {
        CompletableFuture<List<UploadResult>> completionFuture = new CompletableFuture<>();
        store.upload(new UploadTask(getChanges(4), completionFuture::complete, (unused, throwable) -> completionFuture.completeExceptionally(throwable)));
        completionFuture.get();
    }
}

Example 2 with UploadTask

use of org.apache.flink.changelog.fs.StateChangeUploader.UploadTask in project flink by apache.

the class FsStateChangelogWriter method persistInternal.

private CompletableFuture<ChangelogStateHandleStreamImpl> persistInternal(SequenceNumber from) throws IOException {
    synchronized (lock) {
        ensureCanPersist(from);
        rollover();
        Map<SequenceNumber, StateChangeSet> toUpload = drainTailMap(notUploaded, from);
        NavigableMap<SequenceNumber, UploadResult> readyToReturn = uploaded.tailMap(from, true);
        LOG.debug("collected readyToReturn: {}, toUpload: {}", readyToReturn, toUpload);
        SequenceNumberRange range = SequenceNumberRange.generic(from, activeSequenceNumber);
        if (range.size() == readyToReturn.size()) {
            checkState(toUpload.isEmpty());
            return completedFuture(buildHandle(keyGroupRange, readyToReturn, 0L));
        } else {
            CompletableFuture<ChangelogStateHandleStreamImpl> future = new CompletableFuture<>();
            uploadCompletionListeners.add(new UploadCompletionListener(keyGroupRange, range, readyToReturn, future));
            if (!toUpload.isEmpty()) {
                uploader.upload(new UploadTask(toUpload.values(), this::handleUploadSuccess, this::handleUploadFailure));
            }
            return future;
        }
    }
}