Examples with OutputStreamBasedPendingFileRecoverable org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverable used on opensource projects

Example 1 with OutputStreamBasedPendingFileRecoverable

use of org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverable in project flink by apache.

the class FileSinkCommittableSerializerMigrationTest method prepareDeserializationPending.

@Test
@Ignore
public void prepareDeserializationPending() throws IOException {
    String scenario = "pending";
    java.nio.file.Path path = resolveVersionPath(CURRENT_VERSION, scenario);
    BucketWriter<String, String> bucketWriter = createBucketWriter();
    RecoverableWriter writer = FileSystem.getLocalFileSystem().createRecoverableWriter();
    FileSinkCommittableSerializer serializer = new FileSinkCommittableSerializer(bucketWriter.getProperties().getPendingFileRecoverableSerializer(), bucketWriter.getProperties().getInProgressFileRecoverableSerializer());
    RecoverableFsDataOutputStream outputStream = writer.open(new Path(path.resolve("content").toString()));
    outputStream.write(PENDING_CONTENT.getBytes(StandardCharsets.UTF_8));
    CommitRecoverable commitRecoverable = outputStream.closeForCommit().getRecoverable();
    OutputStreamBasedPendingFileRecoverable recoverable = new OutputStreamBasedPendingFileRecoverable(commitRecoverable);
    FileSinkCommittable committable = new FileSinkCommittable("0", recoverable);
    byte[] bytes = serializer.serialize(committable);
    Files.write(path.resolve("committable"), bytes);
}

Example 2 with OutputStreamBasedPendingFileRecoverable

use of org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverable in project flink by apache.

the class OutputStreamBasedPartFileRecoverableMigrationTest method testSerializationPending.

@Test
public void testSerializationPending() throws IOException {
    String scenario = "pending";
    java.nio.file.Path path = resolveVersionPath(previousVersion, scenario);
    RecoverableWriter writer = FileSystem.getLocalFileSystem().createRecoverableWriter();
    OutputStreamBasedPendingFileRecoverableSerializer serializer = new OutputStreamBasedPendingFileRecoverableSerializer(writer.getCommitRecoverableSerializer());
    PendingFileRecoverable recoverable = serializer.deserialize(previousVersion, Files.readAllBytes(path.resolve("recoverable")));
    Assert.assertTrue(recoverable instanceof OutputStreamBasedPendingFileRecoverable);
    // make sure the CommitRecoverable is valid
    writer.recoverForCommit(((OutputStreamBasedPendingFileRecoverable) recoverable).getCommitRecoverable());
}

Example 3 with OutputStreamBasedPendingFileRecoverable

use of org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverable in project flink by apache.

the class OutputStreamBasedPartFileRecoverableMigrationTest method prepareDeserializationPending.

@Test
@Ignore
public void prepareDeserializationPending() throws IOException {
    String scenario = "pending";
    java.nio.file.Path path = resolveVersionPath(CURRENT_VERSION, scenario);
    RecoverableWriter writer = FileSystem.getLocalFileSystem().createRecoverableWriter();
    OutputStreamBasedPendingFileRecoverableSerializer serializer = new OutputStreamBasedPendingFileRecoverableSerializer(writer.getCommitRecoverableSerializer());
    RecoverableFsDataOutputStream outputStream = writer.open(new Path(path.resolve("content").toString()));
    outputStream.write(PENDING_CONTENT.getBytes(StandardCharsets.UTF_8));
    CommitRecoverable commitRecoverable = outputStream.closeForCommit().getRecoverable();
    OutputStreamBasedPendingFileRecoverable recoverable = new OutputStreamBasedPendingFileRecoverable(commitRecoverable);
    byte[] bytes = serializer.serialize(recoverable);
    Files.write(path.resolve("recoverable"), bytes);
}