Examples with BucketStatePathResolver org.apache.flink.streaming.api.functions.sink.filesystem.BucketStatePathResolver used on opensource projects

Example 1 with BucketStatePathResolver

use of org.apache.flink.streaming.api.functions.sink.filesystem.BucketStatePathResolver in project flink by apache.

the class FileWriterBucketStateSerializerMigrationTest method testSerializationOnlyInProgress.

@Test
public void testSerializationOnlyInProgress() throws IOException {
    final String scenarioName = "only-in-progress";
    final BucketStatePathResolver pathResolver = new BucketStatePathResolver(BASE_PATH, previousVersion);
    final java.nio.file.Path outputPath = pathResolver.getOutputPath(scenarioName);
    final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());
    final FileWriterBucketState recoveredState = readBucketState(scenarioName, previousVersion);
    final FileWriterBucket<String> bucket = restoreBucket(recoveredState);
    Assert.assertEquals(testBucketPath, bucket.getBucketPath());
    // check restore the correct in progress file writer
    Assert.assertEquals(8, bucket.getInProgressPart().getSize());
    long numFiles = Files.list(Paths.get(testBucketPath.toString())).map(file -> {
        assertThat(file.getFileName().toString(), startsWith(".part-0-0.inprogress"));
        return 1;
    }).count();
    assertThat(numFiles, is(1L));
}

Example 2 with BucketStatePathResolver

use of org.apache.flink.streaming.api.functions.sink.filesystem.BucketStatePathResolver in project flink by apache.

the class FileWriterBucketStateSerializerMigrationTest method readBucketStateFromTemplate.

private static FileWriterBucketState readBucketStateFromTemplate(final String scenarioName, final int version) throws IOException {
    final BucketStatePathResolver pathResolver = new BucketStatePathResolver(BASE_PATH, version);
    final java.nio.file.Path scenarioPath = pathResolver.getResourcePath(scenarioName);
    // clear the scenario files first
    FileUtils.deleteDirectory(scenarioPath.toFile());
    // prepare the scenario files
    FileUtils.copy(new Path(scenarioPath.toString() + "-template"), new Path(scenarioPath.toString()), false);
    return readBucketState(scenarioName, version);
}

Example 3 with BucketStatePathResolver

use of org.apache.flink.streaming.api.functions.sink.filesystem.BucketStatePathResolver in project flink by apache.

the class FileWriterBucketStateSerializerMigrationTest method testDeserializationFull.

private void testDeserializationFull(final boolean withInProgress, final String scenarioName) throws IOException, InterruptedException {
    final BucketStatePathResolver pathResolver = new BucketStatePathResolver(BASE_PATH, previousVersion);
    try {
        final java.nio.file.Path outputPath = pathResolver.getOutputPath(scenarioName);
        final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());
        // restore the state
        final FileWriterBucketState recoveredState = readBucketStateFromTemplate(scenarioName, previousVersion);
        final int noOfPendingCheckpoints = 5;
        // there are 5 checkpoint does not complete.
        final Map<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverables = recoveredState.getPendingFileRecoverablesPerCheckpoint();
        Assert.assertEquals(5L, pendingFileRecoverables.size());
        final Set<String> beforeRestorePaths = Files.list(outputPath.resolve(BUCKET_ID)).map(file -> file.getFileName().toString()).collect(Collectors.toSet());
        // before retsoring all file has "inprogress"
        for (int i = 0; i < noOfPendingCheckpoints; i++) {
            final String part = ".part-0-" + i + ".inprogress";
            assertThat(beforeRestorePaths, hasItem(startsWith(part)));
        }
        final FileWriterBucket<String> bucket = restoreBucket(recoveredState);
        Assert.assertEquals(testBucketPath, bucket.getBucketPath());
        Assert.assertEquals(noOfPendingCheckpoints, bucket.getPendingFiles().size());
        // simulates we commit the recovered pending files on the first checkpoint
        bucket.snapshotState();
        Collection<CommitRequest<FileSinkCommittable>> committables = bucket.prepareCommit(false).stream().map(MockCommitRequest::new).collect(Collectors.toList());
        FileCommitter committer = new FileCommitter(createBucketWriter());
        committer.commit(committables);
        final Set<String> afterRestorePaths = Files.list(outputPath.resolve(BUCKET_ID)).map(file -> file.getFileName().toString()).collect(Collectors.toSet());
        // there is no "inporgress" in file name for the committed files.
        for (int i = 0; i < noOfPendingCheckpoints; i++) {
            final String part = "part-0-" + i;
            assertThat(afterRestorePaths, hasItem(part));
            afterRestorePaths.remove(part);
        }
        if (withInProgress) {
            // only the in-progress must be left
            assertThat(afterRestorePaths, iterableWithSize(1));
            // verify that the in-progress file is still there
            assertThat(afterRestorePaths, hasItem(startsWith(".part-0-" + noOfPendingCheckpoints + ".inprogress")));
        } else {
            assertThat(afterRestorePaths, empty());
        }
    } finally {
        FileUtils.deleteDirectory(pathResolver.getResourcePath(scenarioName).toFile());
    }
}

Example 4 with BucketStatePathResolver

use of org.apache.flink.streaming.api.functions.sink.filesystem.BucketStatePathResolver in project flink by apache.

the class FileWriterBucketStateSerializerMigrationTest method testSerializationEmpty.

@Test
public void testSerializationEmpty() throws IOException {
    final String scenarioName = "empty";
    final BucketStatePathResolver pathResolver = new BucketStatePathResolver(BASE_PATH, previousVersion);
    final java.nio.file.Path outputPath = pathResolver.getOutputPath(scenarioName);
    final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());
    final FileWriterBucketState recoveredState = readBucketState(scenarioName, previousVersion);
    final FileWriterBucket<String> bucket = restoreBucket(recoveredState);
    Assert.assertEquals(testBucketPath, bucket.getBucketPath());
    Assert.assertNull(bucket.getInProgressPart());
    Assert.assertTrue(bucket.getPendingFiles().isEmpty());
}

Example 5 with BucketStatePathResolver

use of org.apache.flink.streaming.api.functions.sink.filesystem.BucketStatePathResolver in project flink by apache.

the class FileWriterBucketStateSerializerMigrationTest method readBucketState.

private static FileWriterBucketState readBucketState(final String scenarioName, final int version) throws IOException {
    final BucketStatePathResolver pathResolver = new BucketStatePathResolver(BASE_PATH, version);
    byte[] bytes = Files.readAllBytes(pathResolver.getSnapshotPath(scenarioName));
    return SimpleVersionedSerialization.readVersionAndDeSerialize(bucketStateSerializer(), bytes);
}