Examples with UploadTask org.apache.flink.changelog.fs.StateChangeUploadScheduler.UploadTask used on opensource projects

Example 1 with UploadTask

use of org.apache.flink.changelog.fs.StateChangeUploadScheduler.UploadTask in project flink by splunk.

the class BatchingStateChangeUploadSchedulerTest method testRetry.

/**
 * Test integration with {@link RetryingExecutor}.
 */
@Test
public void testRetry() throws Exception {
    final int maxAttempts = 5;
    try (BatchingStateChangeUploadScheduler store = new BatchingStateChangeUploadScheduler(0, 0, MAX_BYTES_IN_FLIGHT, RetryPolicy.fixed(maxAttempts, 0, 0), new TestingStateChangeUploader() {

        final AtomicInteger currentAttempt = new AtomicInteger(0);

        @Override
        public UploadTasksResult upload(Collection<UploadTask> tasks) throws IOException {
            for (UploadTask uploadTask : tasks) {
                if (currentAttempt.getAndIncrement() < maxAttempts - 1) {
                    throw new IOException();
                } else {
                    uploadTask.complete(emptyList());
                }
            }
            return null;
        }
    }, new DirectScheduledExecutorService(), new RetryingExecutor(new DirectScheduledExecutorService(), createUnregisteredChangelogStorageMetricGroup().getAttemptsPerUpload()), createUnregisteredChangelogStorageMetricGroup())) {
        CompletableFuture<List<UploadResult>> completionFuture = new CompletableFuture<>();
        store.upload(new UploadTask(getChanges(4), completionFuture::complete, (unused, throwable) -> completionFuture.completeExceptionally(throwable)));
        completionFuture.get();
    }
}

Example 2 with UploadTask

use of org.apache.flink.changelog.fs.StateChangeUploadScheduler.UploadTask in project flink by splunk.

the class BatchingStateChangeUploadSchedulerTest method testRetryOnTimeout.

@Test
public void testRetryOnTimeout() throws Exception {
    int numAttempts = 3;
    AtomicReference<List<SequenceNumber>> failed = new AtomicReference<>(emptyList());
    AtomicReference<List<UploadResult>> succeeded = new AtomicReference<>(emptyList());
    UploadTask upload = new UploadTask(getChanges(4), succeeded::set, (sqn, error) -> failed.set(sqn));
    ManuallyTriggeredScheduledExecutorService executorService = new ManuallyTriggeredScheduledExecutorService();
    BlockingUploader uploader = new BlockingUploader();
    try (BatchingStateChangeUploadScheduler store = scheduler(numAttempts, executorService, uploader, 10)) {
        store.upload(upload);
        Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5));
        while (uploader.getUploadsCount() < numAttempts - 1 && deadline.hasTimeLeft()) {
            executorService.triggerScheduledTasks();
            executorService.triggerAll();
            Thread.sleep(10);
        }
        uploader.unblock();
        while (!upload.finished.get() && deadline.hasTimeLeft()) {
            executorService.triggerScheduledTasks();
            executorService.triggerAll();
            Thread.sleep(10);
        }
    }
    assertTrue(upload.finished.get());
    assertEquals(upload.changeSets.stream().map(StateChangeSet::getSequenceNumber).collect(Collectors.toSet()), succeeded.get().stream().map(UploadResult::getSequenceNumber).collect(Collectors.toSet()));
    assertTrue(failed.get().isEmpty());
}

Example 3 with UploadTask

use of org.apache.flink.changelog.fs.StateChangeUploadScheduler.UploadTask in project flink by splunk.

the class TestingStateChangeUploader method upload.

@Override
public UploadTasksResult upload(Collection<UploadTask> tasks) throws IOException {
    for (UploadTask uploadTask : tasks) {
        this.uploaded.addAll(uploadTask.changeSets);
        this.tasks.add(uploadTask);
    }
    return new UploadTasksResult(emptyMap(), new ByteStreamStateHandle("", new byte[0]));
}

Example 4 with UploadTask

use of org.apache.flink.changelog.fs.StateChangeUploadScheduler.UploadTask in project flink by splunk.

the class StateChangeFsUploader method upload.

private UploadTasksResult upload(Path path, Collection<UploadTask> tasks) throws IOException {
    boolean wrappedStreamClosed = false;
    FSDataOutputStream fsStream = fileSystem.create(path, NO_OVERWRITE);
    try {
        fsStream.write(compression ? 1 : 0);
        try (OutputStreamWithPos stream = wrap(fsStream)) {
            final Map<UploadTask, Map<StateChangeSet, Long>> tasksOffsets = new HashMap<>();
            for (UploadTask task : tasks) {
                tasksOffsets.put(task, format.write(stream, task.changeSets));
            }
            FileStateHandle handle = new FileStateHandle(path, stream.getPos());
            // otherwise JM may receive invalid handles
            return new UploadTasksResult(tasksOffsets, handle);
        } finally {
            wrappedStreamClosed = true;
        }
    } finally {
        if (!wrappedStreamClosed) {
            fsStream.close();
        }
    }
}

Example 5 with UploadTask

use of org.apache.flink.changelog.fs.StateChangeUploadScheduler.UploadTask in project flink by splunk.

the class StateChangeFsUploader method upload.

public UploadTasksResult upload(Collection<UploadTask> tasks) throws IOException {
    final String fileName = generateFileName();
    LOG.debug("upload {} tasks to {}", tasks.size(), fileName);
    Path path = new Path(basePath, fileName);
    try {
        return uploadWithMetrics(path, tasks);
    } catch (IOException e) {
        metrics.getUploadFailuresCounter().inc();
        try (Closer closer = Closer.create()) {
            closer.register(() -> {
                throw e;
            });
            tasks.forEach(cs -> closer.register(() -> cs.fail(e)));
            closer.register(() -> fileSystem.delete(path, true));
        }
    }
    // closer above throws an exception
    return null;
}