Examples with InProgressFileRecoverable org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.InProgressFileRecoverable used on opensource projects

Example 1 with InProgressFileRecoverable

use of org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.InProgressFileRecoverable in project flink by apache.

the class FileWriterBucket method snapshotState.

FileWriterBucketState snapshotState() throws IOException {
    InProgressFileWriter.InProgressFileRecoverable inProgressFileRecoverable = null;
    long inProgressFileCreationTime = Long.MAX_VALUE;
    if (inProgressPart != null) {
        inProgressFileRecoverable = inProgressPart.persist();
        inProgressFileToCleanup = inProgressFileRecoverable;
        inProgressFileCreationTime = inProgressPart.getCreationTime();
    }
    return new FileWriterBucketState(bucketId, bucketPath, inProgressFileCreationTime, inProgressFileRecoverable);
}

Example 2 with InProgressFileRecoverable

use of org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.InProgressFileRecoverable in project flink by apache.

the class OutputStreamBasedPartFileRecoverableMigrationTest method testSerializationInProgress.

@Test
public void testSerializationInProgress() throws IOException {
    String scenario = "in-progress";
    java.nio.file.Path path = resolveVersionPath(previousVersion, scenario);
    RecoverableWriter writer = FileSystem.getLocalFileSystem().createRecoverableWriter();
    OutputStreamBasedInProgressFileRecoverableSerializer serializer = new OutputStreamBasedInProgressFileRecoverableSerializer(writer.getResumeRecoverableSerializer());
    InProgressFileRecoverable recoverable = serializer.deserialize(previousVersion, Files.readAllBytes(path.resolve("recoverable")));
    Assert.assertTrue(recoverable instanceof OutputStreamBasedInProgressFileRecoverable);
    // make sure the ResumeRecoverable is valid
    writer.recover(((OutputStreamBasedInProgressFileRecoverable) recoverable).getResumeRecoverable());
}

Example 3 with InProgressFileRecoverable

use of org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.InProgressFileRecoverable in project flink by apache.

the class CompactorOperatorTest method createTestBucketWriter.

private BucketWriter<?, String> createTestBucketWriter() {
    return new BucketWriter<Integer, String>() {

        @Override
        public InProgressFileWriter<Integer, String> openNewInProgressFile(String bucketId, Path path, long creationTime) throws IOException {
            return new InProgressFileWriter<Integer, String>() {

                BufferedWriter writer;

                long size = 0L;

                @Override
                public void write(Integer element, long currentTime) throws IOException {
                    if (writer == null) {
                        writer = new BufferedWriter(new FileWriter(path.toString()));
                    }
                    writer.write(element);
                    size += 1;
                }

                @Override
                public InProgressFileRecoverable persist() throws IOException {
                    return new TestInProgressFileRecoverable(path, size);
                }

                @Override
                public PendingFileRecoverable closeForCommit() throws IOException {
                    return new TestPendingFileRecoverable(path, size);
                }

                @Override
                public void dispose() {
                }

                @Override
                public String getBucketId() {
                    return bucketId;
                }

                @Override
                public long getCreationTime() {
                    return 0;
                }

                @Override
                public long getSize() throws IOException {
                    return size;
                }

                @Override
                public long getLastUpdateTime() {
                    return 0;
                }
            };
        }

        @Override
        public InProgressFileWriter<Integer, String> resumeInProgressFileFrom(String s, InProgressFileRecoverable inProgressFileSnapshot, long creationTime) throws IOException {
            return null;
        }

        @Override
        public WriterProperties getProperties() {
            return null;
        }

        @Override
        public PendingFile recoverPendingFile(PendingFileRecoverable pendingFileRecoverable) throws IOException {
            return new PendingFile() {

                @Override
                public void commit() throws IOException {
                    TestPendingFileRecoverable testRecoverable = (TestPendingFileRecoverable) pendingFileRecoverable;
                    if (testRecoverable.getPath() != null) {
                        if (!testRecoverable.getPath().equals(testRecoverable.getUncommittedPath())) {
                            testRecoverable.getPath().getFileSystem().rename(testRecoverable.getUncommittedPath(), testRecoverable.getPath());
                        }
                    }
                }

                @Override
                public void commitAfterRecovery() throws IOException {
                    commit();
                }
            };
        }

        @Override
        public boolean cleanupInProgressFileRecoverable(InProgressFileRecoverable inProgressFileRecoverable) throws IOException {
            return false;
        }

        @Override
        public CompactingFileWriter openNewCompactingFile(CompactingFileWriter.Type type, String bucketId, Path path, long creationTime) throws IOException {
            if (type == CompactingFileWriter.Type.RECORD_WISE) {
                return openNewInProgressFile(bucketId, path, creationTime);
            } else {
                FileOutputStream fileOutputStream = new FileOutputStream(path.toString());
                return new OutputStreamBasedCompactingFileWriter() {

                    @Override
                    public OutputStream asOutputStream() throws IOException {
                        return fileOutputStream;
                    }

                    @Override
                    public PendingFileRecoverable closeForCommit() throws IOException {
                        fileOutputStream.flush();
                        return new TestPendingFileRecoverable(path, fileOutputStream.getChannel().position());
                    }
                };
            }
        }
    };
}

Example 4 with InProgressFileRecoverable

use of org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.InProgressFileRecoverable in project flink by apache.

the class FileWriterBucketStateSerializer method serializeV3.

private void serializeV3(FileWriterBucketState state, DataOutputView dataOutputView) throws IOException {
    SimpleVersionedSerialization.writeVersionAndSerialize(SimpleVersionedStringSerializer.INSTANCE, state.getBucketId(), dataOutputView);
    dataOutputView.writeUTF(state.getBucketPath().toString());
    dataOutputView.writeLong(state.getInProgressFileCreationTime());
    // put the current open part file
    if (state.hasInProgressFileRecoverable()) {
        InProgressFileRecoverable inProgressFileRecoverable = state.getInProgressFileRecoverable();
        dataOutputView.writeBoolean(true);
        SimpleVersionedSerialization.writeVersionAndSerialize(inProgressFileRecoverableSerializer, inProgressFileRecoverable, dataOutputView);
    } else {
        dataOutputView.writeBoolean(false);
    }
}

Example 5 with InProgressFileRecoverable

use of org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.InProgressFileRecoverable in project flink by apache.

the class FileWriterBucketStateSerializer method internalDeserialize.

private FileWriterBucketState internalDeserialize(DataInputView dataInputView, FunctionWithException<DataInputView, InProgressFileRecoverable, IOException> inProgressFileParser, @Nullable BiFunctionWithException<Integer, byte[], PendingFileRecoverable, IOException> pendingFileParser) throws IOException {
    String bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(SimpleVersionedStringSerializer.INSTANCE, dataInputView);
    String bucketPathStr = dataInputView.readUTF();
    long creationTime = dataInputView.readLong();
    // then get the current resumable stream
    InProgressFileRecoverable current = null;
    if (dataInputView.readBoolean()) {
        current = inProgressFileParser.apply(dataInputView);
    }
    HashMap<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverablesPerCheckpoint = new HashMap<>();
    if (pendingFileParser != null) {
        final int pendingFileRecoverableSerializerVersion = dataInputView.readInt();
        final int numCheckpoints = dataInputView.readInt();
        for (int i = 0; i < numCheckpoints; i++) {
            final long checkpointId = dataInputView.readLong();
            final int numOfPendingFileRecoverables = dataInputView.readInt();
            final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverables = new ArrayList<>(numOfPendingFileRecoverables);
            for (int j = 0; j < numOfPendingFileRecoverables; j++) {
                final byte[] bytes = new byte[dataInputView.readInt()];
                dataInputView.readFully(bytes);
                pendingFileRecoverables.add(pendingFileParser.apply(pendingFileRecoverableSerializerVersion, bytes));
            }
            pendingFileRecoverablesPerCheckpoint.put(checkpointId, pendingFileRecoverables);
        }
    }
    return new FileWriterBucketState(bucketId, new Path(bucketPathStr), creationTime, current, pendingFileRecoverablesPerCheckpoint);
}