Examples with CommitRequest org.apache.flink.api.connector.sink2.Committer.CommitRequest used on opensource projects

Example 1 with CommitRequest

use of org.apache.flink.api.connector.sink2.Committer.CommitRequest in project flink by apache.

the class FileWriterBucketStateSerializerMigrationTest method testDeserializationFull.

private void testDeserializationFull(final boolean withInProgress, final String scenarioName) throws IOException, InterruptedException {
    final BucketStatePathResolver pathResolver = new BucketStatePathResolver(BASE_PATH, previousVersion);
    try {
        final java.nio.file.Path outputPath = pathResolver.getOutputPath(scenarioName);
        final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());
        // restore the state
        final FileWriterBucketState recoveredState = readBucketStateFromTemplate(scenarioName, previousVersion);
        final int noOfPendingCheckpoints = 5;
        // there are 5 checkpoint does not complete.
        final Map<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverables = recoveredState.getPendingFileRecoverablesPerCheckpoint();
        Assert.assertEquals(5L, pendingFileRecoverables.size());
        final Set<String> beforeRestorePaths = Files.list(outputPath.resolve(BUCKET_ID)).map(file -> file.getFileName().toString()).collect(Collectors.toSet());
        // before retsoring all file has "inprogress"
        for (int i = 0; i < noOfPendingCheckpoints; i++) {
            final String part = ".part-0-" + i + ".inprogress";
            assertThat(beforeRestorePaths, hasItem(startsWith(part)));
        }
        final FileWriterBucket<String> bucket = restoreBucket(recoveredState);
        Assert.assertEquals(testBucketPath, bucket.getBucketPath());
        Assert.assertEquals(noOfPendingCheckpoints, bucket.getPendingFiles().size());
        // simulates we commit the recovered pending files on the first checkpoint
        bucket.snapshotState();
        Collection<CommitRequest<FileSinkCommittable>> committables = bucket.prepareCommit(false).stream().map(MockCommitRequest::new).collect(Collectors.toList());
        FileCommitter committer = new FileCommitter(createBucketWriter());
        committer.commit(committables);
        final Set<String> afterRestorePaths = Files.list(outputPath.resolve(BUCKET_ID)).map(file -> file.getFileName().toString()).collect(Collectors.toSet());
        // there is no "inporgress" in file name for the committed files.
        for (int i = 0; i < noOfPendingCheckpoints; i++) {
            final String part = "part-0-" + i;
            assertThat(afterRestorePaths, hasItem(part));
            afterRestorePaths.remove(part);
        }
        if (withInProgress) {
            // only the in-progress must be left
            assertThat(afterRestorePaths, iterableWithSize(1));
            // verify that the in-progress file is still there
            assertThat(afterRestorePaths, hasItem(startsWith(".part-0-" + noOfPendingCheckpoints + ".inprogress")));
        } else {
            assertThat(afterRestorePaths, empty());
        }
    } finally {
        FileUtils.deleteDirectory(pathResolver.getResourcePath(scenarioName).toFile());
    }
}

Example 2 with CommitRequest

use of org.apache.flink.api.connector.sink2.Committer.CommitRequest in project flink by apache.

the class FileCommitterTest method testCommitMultiple.

@Test
public void testCommitMultiple() throws Exception {
    StubBucketWriter stubBucketWriter = new StubBucketWriter();
    FileCommitter fileCommitter = new FileCommitter(stubBucketWriter);
    Collection<CommitRequest<FileSinkCommittable>> committables = Stream.of(new FileSinkCommittable("0", new FileSinkTestUtils.TestPendingFileRecoverable()), new FileSinkCommittable("0", new FileSinkTestUtils.TestPendingFileRecoverable()), new FileSinkCommittable("0", new FileSinkTestUtils.TestInProgressFileRecoverable()), new FileSinkCommittable("0", new FileSinkTestUtils.TestPendingFileRecoverable()), new FileSinkCommittable("0", new FileSinkTestUtils.TestInProgressFileRecoverable())).map(MockCommitRequest::new).collect(Collectors.toList());
    fileCommitter.commit(committables);
    assertEquals(3, stubBucketWriter.getRecoveredPendingFiles().size());
    assertEquals(2, stubBucketWriter.getNumCleanUp());
    stubBucketWriter.getRecoveredPendingFiles().forEach(pendingFile -> assertTrue(pendingFile.isCommitted()));
    assertTrue(committables.stream().allMatch(c -> c.getNumberOfRetries() == 0));
}