Search in sources :

Example 1 with CompactingFileWriter

use of org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter in project flink by apache.

the class CompactorOperatorTest method createTestBucketWriter.

private BucketWriter<?, String> createTestBucketWriter() {
    return new BucketWriter<Integer, String>() {

        @Override
        public InProgressFileWriter<Integer, String> openNewInProgressFile(String bucketId, Path path, long creationTime) throws IOException {
            return new InProgressFileWriter<Integer, String>() {

                BufferedWriter writer;

                long size = 0L;

                @Override
                public void write(Integer element, long currentTime) throws IOException {
                    if (writer == null) {
                        writer = new BufferedWriter(new FileWriter(path.toString()));
                    }
                    writer.write(element);
                    size += 1;
                }

                @Override
                public InProgressFileRecoverable persist() throws IOException {
                    return new TestInProgressFileRecoverable(path, size);
                }

                @Override
                public PendingFileRecoverable closeForCommit() throws IOException {
                    return new TestPendingFileRecoverable(path, size);
                }

                @Override
                public void dispose() {
                }

                @Override
                public String getBucketId() {
                    return bucketId;
                }

                @Override
                public long getCreationTime() {
                    return 0;
                }

                @Override
                public long getSize() throws IOException {
                    return size;
                }

                @Override
                public long getLastUpdateTime() {
                    return 0;
                }
            };
        }

        @Override
        public InProgressFileWriter<Integer, String> resumeInProgressFileFrom(String s, InProgressFileRecoverable inProgressFileSnapshot, long creationTime) throws IOException {
            return null;
        }

        @Override
        public WriterProperties getProperties() {
            return null;
        }

        @Override
        public PendingFile recoverPendingFile(PendingFileRecoverable pendingFileRecoverable) throws IOException {
            return new PendingFile() {

                @Override
                public void commit() throws IOException {
                    TestPendingFileRecoverable testRecoverable = (TestPendingFileRecoverable) pendingFileRecoverable;
                    if (testRecoverable.getPath() != null) {
                        if (!testRecoverable.getPath().equals(testRecoverable.getUncommittedPath())) {
                            testRecoverable.getPath().getFileSystem().rename(testRecoverable.getUncommittedPath(), testRecoverable.getPath());
                        }
                    }
                }

                @Override
                public void commitAfterRecovery() throws IOException {
                    commit();
                }
            };
        }

        @Override
        public boolean cleanupInProgressFileRecoverable(InProgressFileRecoverable inProgressFileRecoverable) throws IOException {
            return false;
        }

        @Override
        public CompactingFileWriter openNewCompactingFile(CompactingFileWriter.Type type, String bucketId, Path path, long creationTime) throws IOException {
            if (type == CompactingFileWriter.Type.RECORD_WISE) {
                return openNewInProgressFile(bucketId, path, creationTime);
            } else {
                FileOutputStream fileOutputStream = new FileOutputStream(path.toString());
                return new OutputStreamBasedCompactingFileWriter() {

                    @Override
                    public OutputStream asOutputStream() throws IOException {
                        return fileOutputStream;
                    }

                    @Override
                    public PendingFileRecoverable closeForCommit() throws IOException {
                        fileOutputStream.flush();
                        return new TestPendingFileRecoverable(path, fileOutputStream.getChannel().position());
                    }
                };
            }
        }
    };
}
Also used : Path(org.apache.flink.core.fs.Path) TestPendingFileRecoverable(org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestPendingFileRecoverable) OutputStreamBasedCompactingFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedCompactingFileWriter) InProgressFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter) OutputStreamBasedCompactingFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedCompactingFileWriter) FileWriter(java.io.FileWriter) CompactingFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter) InProgressFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter) TestInProgressFileRecoverable(org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestInProgressFileRecoverable) BufferedWriter(java.io.BufferedWriter) TestInProgressFileRecoverable(org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestInProgressFileRecoverable) InProgressFileRecoverable(org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.InProgressFileRecoverable) BucketWriter(org.apache.flink.streaming.api.functions.sink.filesystem.BucketWriter) FileOutputStream(java.io.FileOutputStream) PendingFileRecoverable(org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.PendingFileRecoverable) TestPendingFileRecoverable(org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestPendingFileRecoverable)

Example 2 with CompactingFileWriter

use of org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter in project flink by apache.

the class CompactService method compact.

@SuppressWarnings({ "rawtypes", "unchecked" })
private Iterable<FileSinkCommittable> compact(CompactorRequest request) throws Exception {
    List<FileSinkCommittable> results = new ArrayList<>(request.getCommittableToPassthrough());
    List<Path> compactingFiles = getCompactingPath(request);
    if (compactingFiles.isEmpty()) {
        return results;
    }
    Path targetPath = assembleCompactedFilePath(compactingFiles.get(0));
    CompactingFileWriter compactingFileWriter = bucketWriter.openNewCompactingFile(compactingWriterType, request.getBucketId(), targetPath, System.currentTimeMillis());
    if (compactingWriterType == Type.RECORD_WISE) {
        ((RecordWiseFileCompactor) fileCompactor).compact(compactingFiles, ((RecordWiseCompactingFileWriter) compactingFileWriter)::write);
    } else if (compactingWriterType == CompactingFileWriter.Type.OUTPUT_STREAM) {
        ((OutputStreamBasedFileCompactor) fileCompactor).compact(compactingFiles, ((OutputStreamBasedCompactingFileWriter) compactingFileWriter).asOutputStream());
    }
    PendingFileRecoverable compactedPendingFile = compactingFileWriter.closeForCommit();
    FileSinkCommittable compacted = new FileSinkCommittable(request.getBucketId(), compactedPendingFile);
    results.add(compacted);
    for (Path f : compactingFiles) {
        // cleanup compacted files
        results.add(new FileSinkCommittable(request.getBucketId(), f));
    }
    return results;
}
Also used : Path(org.apache.flink.core.fs.Path) OutputStreamBasedCompactingFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedCompactingFileWriter) RecordWiseFileCompactor(org.apache.flink.connector.file.sink.compactor.RecordWiseFileCompactor) ArrayList(java.util.ArrayList) PendingFileRecoverable(org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.PendingFileRecoverable) FileSinkCommittable(org.apache.flink.connector.file.sink.FileSinkCommittable) CompactingFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter) OutputStreamBasedCompactingFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedCompactingFileWriter) RecordWiseCompactingFileWriter(org.apache.flink.streaming.api.functions.sink.filesystem.RecordWiseCompactingFileWriter)

Aggregations

Path (org.apache.flink.core.fs.Path)2 CompactingFileWriter (org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter)2 PendingFileRecoverable (org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.PendingFileRecoverable)2 OutputStreamBasedCompactingFileWriter (org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedCompactingFileWriter)2 BufferedWriter (java.io.BufferedWriter)1 FileOutputStream (java.io.FileOutputStream)1 FileWriter (java.io.FileWriter)1 ArrayList (java.util.ArrayList)1 FileSinkCommittable (org.apache.flink.connector.file.sink.FileSinkCommittable)1 RecordWiseFileCompactor (org.apache.flink.connector.file.sink.compactor.RecordWiseFileCompactor)1 TestInProgressFileRecoverable (org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestInProgressFileRecoverable)1 TestPendingFileRecoverable (org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestPendingFileRecoverable)1 BucketWriter (org.apache.flink.streaming.api.functions.sink.filesystem.BucketWriter)1 InProgressFileWriter (org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter)1 InProgressFileRecoverable (org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.InProgressFileRecoverable)1 RecordWiseCompactingFileWriter (org.apache.flink.streaming.api.functions.sink.filesystem.RecordWiseCompactingFileWriter)1