Search in sources :

Example 1 with TestPendingFileRecoverable

use of org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestPendingFileRecoverable in project flink by apache.

the class CompactorOperatorTest method createTestBucketWriter.

private BucketWriter<?, String> createTestBucketWriter() {
    return new BucketWriter<Integer, String>() {

        @Override
        public InProgressFileWriter<Integer, String> openNewInProgressFile(String bucketId, Path path, long creationTime) throws IOException {
            return new InProgressFileWriter<Integer, String>() {

                BufferedWriter writer;

                long size = 0L;

                @Override
                public void write(Integer element, long currentTime) throws IOException {
                    if (writer == null) {
                        writer = new BufferedWriter(new FileWriter(path.toString()));
                    }
                    writer.write(element);
                    size += 1;
                }

                @Override
                public InProgressFileRecoverable persist() throws IOException {
                    return new TestInProgressFileRecoverable(path, size);
                }

                @Override
                public PendingFileRecoverable closeForCommit() throws IOException {
                    return new TestPendingFileRecoverable(path, size);
                }

                @Override
                public void dispose() {
                }

                @Override
                public String getBucketId() {
                    return bucketId;
                }

                @Override
                public long getCreationTime() {
                    return 0;
                }

                @Override
                public long getSize() throws IOException {
                    return size;
                }

                @Override
                public long getLastUpdateTime() {
                    return 0;
                }
            };
        }

        @Override
        public InProgressFileWriter<Integer, String> resumeInProgressFileFrom(String s, InProgressFileRecoverable inProgressFileSnapshot, long creationTime) throws IOException {
            return null;
        }

        @Override
        public WriterProperties getProperties() {
            return null;
        }

        @Override
        public PendingFile recoverPendingFile(PendingFileRecoverable pendingFileRecoverable) throws IOException {
            return new PendingFile() {

                @Override
                public void commit() throws IOException {
                    TestPendingFileRecoverable testRecoverable = (TestPendingFileRecoverable) pendingFileRecoverable;
                    if (testRecoverable.getPath() != null) {
                        if (!testRecoverable.getPath().equals(testRecoverable.getUncommittedPath())) {
                            testRecoverable.getPath().getFileSystem().rename(testRecoverable.getUncommittedPath(), testRecoverable.getPath());
                        }
                    }
                }

                @Override
                public void commitAfterRecovery() throws IOException {
                    commit();
                }
            };
        }

        @Override
        public boolean cleanupInProgressFileRecoverable(InProgressFileRecoverable inProgressFileRecoverable) throws IOException {
            return false;
        }

        @Override
        public CompactingFileWriter openNewCompactingFile(CompactingFileWriter.Type type, String bucketId, Path path, long creationTime) throws IOException {
            if (type == CompactingFileWriter.Type.RECORD_WISE) {
                return openNewInProgressFile(bucketId, path, creationTime);
            } else {
                FileOutputStream fileOutputStream = new FileOutputStream(path.toString());
                return new OutputStreamBasedCompactingFileWriter() {

                    @Override
                    public OutputStream asOutputStream() throws IOException {
                        return fileOutputStream;
                    }

                    @Override
                    public PendingFileRecoverable closeForCommit() throws IOException {
                        fileOutputStream.flush();
                        return new TestPendingFileRecoverable(path, fileOutputStream.getChannel().position());
                    }
                };
            }
        }
    };
}
Also used : Path(org.apache.flink.core.fs.Path) TestPendingFileRecoverable(org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestPendingFileRecoverable) OutputStreamBasedCompactingFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedCompactingFileWriter) InProgressFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter) OutputStreamBasedCompactingFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedCompactingFileWriter) FileWriter(java.io.FileWriter) CompactingFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter) InProgressFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter) TestInProgressFileRecoverable(org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestInProgressFileRecoverable) BufferedWriter(java.io.BufferedWriter) TestInProgressFileRecoverable(org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestInProgressFileRecoverable) InProgressFileRecoverable(org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.InProgressFileRecoverable) BucketWriter(org.apache.flink.streaming.api.functions.sink.filesystem.BucketWriter) FileOutputStream(java.io.FileOutputStream) PendingFileRecoverable(org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.PendingFileRecoverable) TestPendingFileRecoverable(org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestPendingFileRecoverable)

Aggregations

BufferedWriter (java.io.BufferedWriter)1 FileOutputStream (java.io.FileOutputStream)1 FileWriter (java.io.FileWriter)1 TestInProgressFileRecoverable (org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestInProgressFileRecoverable)1 TestPendingFileRecoverable (org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestPendingFileRecoverable)1 Path (org.apache.flink.core.fs.Path)1 BucketWriter (org.apache.flink.streaming.api.functions.sink.filesystem.BucketWriter)1 CompactingFileWriter (org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter)1 InProgressFileWriter (org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter)1 InProgressFileRecoverable (org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.InProgressFileRecoverable)1 PendingFileRecoverable (org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.PendingFileRecoverable)1 OutputStreamBasedCompactingFileWriter (org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedCompactingFileWriter)1