Examples with ResumeRecoverable org.apache.flink.core.fs.RecoverableWriter.ResumeRecoverable used on opensource projects

Example 1 with ResumeRecoverable

use of org.apache.flink.core.fs.RecoverableWriter.ResumeRecoverable in project flink by apache.

the class OutputStreamBasedPartFileRecoverableMigrationTest method prepareDeserializationInProgress.

@Test
@Ignore
public void prepareDeserializationInProgress() throws IOException {
    String scenario = "in-progress";
    java.nio.file.Path path = resolveVersionPath(CURRENT_VERSION, scenario);
    RecoverableWriter writer = FileSystem.getLocalFileSystem().createRecoverableWriter();
    OutputStreamBasedInProgressFileRecoverableSerializer serializer = new OutputStreamBasedInProgressFileRecoverableSerializer(writer.getResumeRecoverableSerializer());
    RecoverableFsDataOutputStream outputStream = writer.open(new Path(path.resolve("content").toString()));
    outputStream.write(IN_PROGRESS_CONTENT.getBytes(StandardCharsets.UTF_8));
    ResumeRecoverable resumeRecoverable = outputStream.persist();
    OutputStreamBasedInProgressFileRecoverable recoverable = new OutputStreamBasedInProgressFileRecoverable(resumeRecoverable);
    byte[] bytes = serializer.serialize(recoverable);
    Files.write(path.resolve("recoverable"), bytes);
}

Example 2 with ResumeRecoverable

use of org.apache.flink.core.fs.RecoverableWriter.ResumeRecoverable in project flink by apache.

the class HadoopViewFileSystemTruncateTest method testViewFileSystemRecoverWorks.

@Test
public void testViewFileSystemRecoverWorks() throws IOException {
    final org.apache.flink.core.fs.Path testPath = new org.apache.flink.core.fs.Path(fSystem.getUri() + "mountOnNn1/test-1");
    final String expectedContent = "test_line";
    final RecoverableWriter writer = fSystem.createRecoverableWriter();
    final RecoverableFsDataOutputStream streamUnderTest = getOpenStreamToFileWithContent(writer, testPath, expectedContent);
    final ResumeRecoverable resumeRecover = streamUnderTest.persist();
    final RecoverableFsDataOutputStream recover = writer.recover(resumeRecover);
    final RecoverableWriter.CommitRecoverable committable = recover.closeForCommit().getRecoverable();
    final RecoverableWriter recoveredWriter = fSystem.createRecoverableWriter();
    recoveredWriter.recoverForCommit(committable).commitAfterRecovery();
    verifyFileContent(testPath, expectedContent);
}

Example 3 with ResumeRecoverable

use of org.apache.flink.core.fs.RecoverableWriter.ResumeRecoverable in project flink by apache.

the class FileSinkCommittableSerializerMigrationTest method prepareDeserializationInProgressToCleanup.

@Test
@Ignore
public void prepareDeserializationInProgressToCleanup() throws IOException {
    String scenario = "in-progress";
    java.nio.file.Path path = resolveVersionPath(CURRENT_VERSION, scenario);
    BucketWriter<String, String> bucketWriter = createBucketWriter();
    RecoverableWriter writer = FileSystem.getLocalFileSystem().createRecoverableWriter();
    FileSinkCommittableSerializer serializer = new FileSinkCommittableSerializer(bucketWriter.getProperties().getPendingFileRecoverableSerializer(), bucketWriter.getProperties().getInProgressFileRecoverableSerializer());
    RecoverableFsDataOutputStream outputStream = writer.open(new Path(path.resolve("content").toString()));
    outputStream.write(IN_PROGRESS_CONTENT.getBytes(StandardCharsets.UTF_8));
    ResumeRecoverable resumeRecoverable = outputStream.persist();
    OutputStreamBasedInProgressFileRecoverable recoverable = new OutputStreamBasedInProgressFileRecoverable(resumeRecoverable);
    FileSinkCommittable committable = new FileSinkCommittable("0", recoverable);
    byte[] bytes = serializer.serialize(committable);
    Files.write(path.resolve("committable"), bytes);
}